jessikitty/drop
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

In-app store, torrential backend, locales (#332)

Browse Source 
* feat: add store nav and fixes

* fix: reduce password requirement & new task error ui

* fix: client webtoken fix

* fix: delta versions and dockerfile

* fix: use setup platforms for filter & display

* fix: setup not accounted when returning valid options

* feat: tighter delta version support

* feat: dl/disk size

* feat: offload manifest generation to torrential

* fix: bump torrential

* feat: remove droplet

* feat: bump torrential

* feat: convert locales
This commit is contained in:
DecDuck
2026-02-06 00:12:24 +11:00
committed by GitHub
parent 837bc6eb1d
commit d234f8df33
82 changed files with 1737 additions and 967 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/api/v1/admin/depot/torrential/manifest.get.ts
-55
View File
@@ -1,55 +0,0 @@
import { ArkErrors, type } from "arktype";
import prisma from "~/server/internal/db/database";
import type { H3Event } from "h3";
import { castManifest } from "~/server/internal/library/manifest";
const AUTHORIZATION_HEADER_PREFIX = "Bearer ";
const Query = type({
version: "string",
});
export async function depotAuthorization(h3: H3Event) {
const authorization = getHeader(h3, "Authorization");
if (!authorization) throw createError({ statusCode: 403 });
if (!authorization.startsWith(AUTHORIZATION_HEADER_PREFIX))
throw createError({ statusCode: 403 });
const key = authorization.slice(AUTHORIZATION_HEADER_PREFIX.length);
const depot = await prisma.depot.findFirst({ where: { key } });
if (!depot) throw createError({ statusCode: 403 });
}
export default defineEventHandler(async (h3) => {
await depotAuthorization(h3);
const query = Query(getQuery(h3));
if (query instanceof ArkErrors)
throw createError({ statusCode: 400, message: query.summary });
const version = await prisma.gameVersion.findUnique({
where: {
versionId: query.version,
},
select: {
dropletManifest: true,
versionPath: true,
game: {
select: {
library: true,
libraryPath: true,
},
},
},
});
if (!version)
throw createError({ statusCode: 404, message: "Game version not found" });
return {
manifest: castManifest(version.dropletManifest),
library: version.game.library,
libraryPath: version.game.libraryPath,
versionPath: version.versionPath,
};
});
server/api/v1/admin/depot/torrential/versions.get.ts
-24
View File
@@ -1,24 +0,0 @@
import prisma from "~/server/internal/db/database";
import { depotAuthorization } from "./manifest.get";
export default defineEventHandler(async (h3) => {
await depotAuthorization(h3);
const games = await prisma.game.findMany({
select: {
id: true,
versions: {
select: {
versionId: true,
},
where: {
versionPath: {
not: null
}
}
},
},
});
return games;
});
server/api/v1/admin/game/[id]/index.get.ts
+4 -5
View File
@@ -1,17 +1,16 @@
import type { GameVersion, Prisma } from "~/prisma/client/client";
import aclManager from "~/server/internal/acls";
import prisma from "~/server/internal/db/database";
import gameSizeManager from "~/server/internal/gamesize";
import type { UnimportedVersionInformation } from "~/server/internal/library";
import libraryManager from "~/server/internal/library";
async function getGameVersionSize<
T extends Omit<GameVersion, "dropletManifest">,
>(gameId: string, version: T) {
const size = await libraryManager.getGameVersionSize(
gameId,
version.versionId,
);
return { ...version, size };
const clientSize = await gameSizeManager.getVersionSize(version.versionId);
const diskSize = await gameSizeManager.getVersionDiskSize(version.versionId);
return { ...version, diskSize, clientSize };
}
export type AdminFetchGameType = Prisma.GameGetPayload<{
server/api/v1/admin/home/index.get.ts
-8
View File
@@ -10,18 +10,10 @@ export default defineEventHandler(async (h3) => {
const sources = await libraryManager.fetchLibraries();
const userStats = await userStatsManager.getUserStats();
const biggestGamesCombined =
await libraryManager.getBiggestGamesCombinedVersions(5);
const biggestGamesLatest =
await libraryManager.getBiggestGamesLatestVersions(5);
return {
gameCount: await prisma.game.count(),
version: systemConfig.getDropVersion(),
userStats,
sources,
biggestGamesLatest,
biggestGamesCombined,
};
});
server/api/v1/admin/import/version/index.post.ts
+6 -1
View File
@@ -50,7 +50,12 @@ export default defineEventHandler(async (h3) => {
where: {
gameId: body.id,
delta: false,
launches: { some: { platform: platformObject.platform } },
OR: [
{ launches: { some: { platform: platformObject.platform } } },
{
setups: { some: { platform: platformObject.platform } },
},
],
},
});
if (validOverlayVersions == 0)
server/api/v1/auth/signin/simple.post.ts
+8 -8
View File
@@ -23,7 +23,7 @@ export default defineEventHandler<{
if (!authManager.getAuthProviders().Simple)
throw createError({
statusCode: 403,
statusMessage: t("errors.auth.method.signinDisabled"),
message: t("errors.auth.method.signinDisabled"),
});
const body = signinValidator(await readBody(h3));
@@ -33,7 +33,7 @@ export default defineEventHandler<{
throw createError({
statusCode: 400,
statusMessage: body.summary,
message: body.summary,
});
}
@@ -57,13 +57,13 @@ export default defineEventHandler<{
if (!authMek)
throw createError({
statusCode: 401,
statusMessage: t("errors.auth.invalidUserOrPass"),
message: t("errors.auth.invalidUserOrPass"),
});
if (!authMek.user.enabled)
throw createError({
statusCode: 403,
statusMessage: t("errors.auth.disabled"),
message: t("errors.auth.disabled"),
});
// LEGACY bcrypt
@@ -74,13 +74,13 @@ export default defineEventHandler<{
if (!hash)
throw createError({
statusCode: 500,
statusMessage: t("errors.auth.invalidPassState"),
message: t("errors.auth.invalidPassState"),
});
if (!(await checkHashBcrypt(body.password, hash)))
throw createError({
statusCode: 401,
statusMessage: t("errors.auth.invalidUserOrPass"),
message: t("errors.auth.invalidUserOrPass"),
});
// TODO: send user to forgot password screen or something to force them to change their password to new system
@@ -101,13 +101,13 @@ export default defineEventHandler<{
if (!hash || typeof hash !== "string")
throw createError({
statusCode: 500,
statusMessage: t("errors.auth.invalidPassState"),
message: t("errors.auth.invalidPassState"),
});
if (!(await checkHashArgon2(body.password, hash)))
throw createError({
statusCode: 401,
statusMessage: t("errors.auth.invalidUserOrPass"),
message: t("errors.auth.invalidUserOrPass"),
});
const result = await sessionHandler.signin(h3, authMek.userId, {
server/api/v1/auth/signup/simple.post.ts
+4 -4
View File
@@ -15,7 +15,7 @@ export const SharedRegisterValidator = type({
const CreateUserValidator = SharedRegisterValidator.and({
invitation: "string",
password: "string >= 14",
password: "string >= 8",
"displayName?": "string | undefined",
}).configure(throwingArktype);
@@ -27,7 +27,7 @@ export default defineEventHandler<{
if (!authManager.getAuthProviders().Simple)
throw createError({
statusCode: 403,
statusMessage: t("errors.auth.method.signinDisabled"),
message: t("errors.auth.method.signinDisabled"),
});
const user = await readValidatedBody(h3, CreateUserValidator);
@@ -38,7 +38,7 @@ export default defineEventHandler<{
if (!invitation)
throw createError({
statusCode: 401,
statusMessage: t("errors.auth.invalidInvite"),
message: t("errors.auth.invalidInvite"),
});
// reuse items from invite
@@ -51,7 +51,7 @@ export default defineEventHandler<{
if (existing > 0)
throw createError({
statusCode: 400,
statusMessage: t("errors.auth.usernameTaken"),
message: t("errors.auth.usernameTaken"),
});
const userId = randomUUID();
server/api/v1/client/game/[id]/version/[versionid]/index.get.ts
+1 -5
View File
@@ -1,6 +1,5 @@
import { defineClientEventHandler } from "~/server/internal/clients/event-handler";
import prisma from "~/server/internal/db/database";
import libraryManager from "~/server/internal/library";
export default defineClientEventHandler(async (h3) => {
const id = getRouterParam(h3, "id");
@@ -57,8 +56,5 @@ export default defineClientEventHandler(async (h3) => {
})),
};
return {
...gameVersionMapped,
size: libraryManager.getGameVersionSize(id, version),
};
return gameVersionMapped;
});
server/api/v1/client/game/versions.get.ts → server/api/v1/client/game/[id]/versions.get.ts
+12 -16
View File
@@ -1,6 +1,7 @@
import type { Platform } from "~/prisma/client/enums";
import { defineClientEventHandler } from "~/server/internal/clients/event-handler";
import prisma from "~/server/internal/db/database";
import type { GameVersionSize } from "~/server/internal/gamesize";
import gameSizeManager from "~/server/internal/gamesize";
type VersionDownloadOption = {
@@ -8,24 +9,23 @@ type VersionDownloadOption = {
displayName?: string | undefined;
versionPath?: string | undefined;
platform: Platform;
size: number;
size: GameVersionSize;
requiredContent: Array<{
gameId: string;
versionId: string;
name: string;
iconObjectId: string;
shortDescription: string;
size: number;
size: GameVersionSize;
}>;
};
export default defineClientEventHandler(async (h3) => {
const query = getQuery(h3);
const id = query.id?.toString();
const id = getRouterParam(h3, "id")!;
if (!id)
throw createError({
statusCode: 400,
statusMessage: "No ID in request query",
statusMessage: "No ID in router params",
});
const rawVersions = await prisma.gameVersion.findMany({
@@ -62,6 +62,7 @@ export default defineClientEventHandler(async (h3) => {
},
},
},
setups: true,
},
});
@@ -73,11 +74,11 @@ export default defineClientEventHandler(async (h3) => {
VersionDownloadOption["requiredContent"]
> = new Map();
for (const launch of v.launches) {
for (const launch of [...v.launches, ...v.setups]) {
if (!platformOptions.has(launch.platform))
platformOptions.set(launch.platform, []);
if (launch.executor) {
if ("executor" in launch && launch.executor) {
const old = platformOptions.get(launch.platform)!;
old.push({
gameId: launch.executor.gameVersion.game.id,
@@ -86,19 +87,14 @@ export default defineClientEventHandler(async (h3) => {
iconObjectId: launch.executor.gameVersion.game.mIconObjectId,
shortDescription:
launch.executor.gameVersion.game.mShortDescription,
size:
(await gameSizeManager.getGameVersionSize(
launch.executor.gameVersion.game.id,
launch.executor.gameVersion.versionId,
)) ?? 0,
size: (await gameSizeManager.getVersionSize(
launch.executor.gameVersion.versionId,
))!,
});
}
}
const size = await gameSizeManager.getGameVersionSize(
v.gameId,
v.versionId,
);
const size = await gameSizeManager.getVersionSize(v.versionId);
return platformOptions
.entries()
server/api/v1/client/user/webtoken.post.ts
+2 -10
View File
@@ -1,29 +1,21 @@
import { APITokenMode } from "~/prisma/client/enums";
import { DateTime } from "luxon";
import type { UserACL } from "~/server/internal/acls";
import { defineClientEventHandler } from "~/server/internal/clients/event-handler";
import prisma from "~/server/internal/db/database";
import { CLIENT_WEBTOKEN_ACLS } from "~/server/plugins/04.auth-init";
export default defineClientEventHandler(
async (h3, { fetchUser, fetchClient, clientId }) => {
const user = await fetchUser();
const client = await fetchClient();
const acls: UserACL = [
"read",
"store:read",
"collections:read",
"object:read",
"settings:read",
];
const token = await prisma.aPIToken.create({
data: {
name: `${client.name} Web Access Token ${DateTime.now().toISO()}`,
clientId,
userId: user.id,
mode: APITokenMode.Client,
acls,
acls: CLIENT_WEBTOKEN_ACLS,
},
});
server/api/v1/collection/index.post.ts
+12 -7
View File
@@ -1,20 +1,25 @@
import { type } from "arktype";
import { readDropValidatedBody, throwingArktype } from "~/server/arktype";
import aclManager from "~/server/internal/acls";
import userLibraryManager from "~/server/internal/userlibrary";
const CreateCollection = type({
name: "string",
}).configure(throwingArktype);
export default defineEventHandler(async (h3) => {
const userId = await aclManager.getUserIdACL(h3, ["collections:read"]);
const userId = await aclManager.getUserIdACL(h3, ["collections:new"]);
if (!userId)
throw createError({
statusCode: 403,
});
const body = await readBody(h3);
const name = body.name;
if (!name)
throw createError({ statusCode: 400, statusMessage: "Requires name" });
const body = await readDropValidatedBody(h3, CreateCollection);
// Create the collection using the manager
const newCollection = await userLibraryManager.collectionCreate(name, userId);
const newCollection = await userLibraryManager.collectionCreate(
body.name,
userId,
);
return newCollection;
});
server/api/v1/games/[id]/index.get.ts
+2 -2
View File
@@ -1,6 +1,6 @@
import aclManager from "~/server/internal/acls";
import prisma from "~/server/internal/db/database";
import libraryManager from "~/server/internal/library";
import gameSizeManager from "~/server/internal/gamesize";
export default defineEventHandler(async (h3) => {
const userId = await aclManager.getUserIdACL(h3, ["store:read"]);
@@ -57,7 +57,7 @@ export default defineEventHandler(async (h3) => {
},
});
const size = await libraryManager.getGameVersionSize(game.id);
const size = (await gameSizeManager.getGameBreakdown(gameId))!;
return { game, rating, size };
});
server/api/v1/store/index.get.ts
+16 -7
View File
@@ -32,6 +32,11 @@ export default defineEventHandler(async (h3) => {
if (options instanceof ArkErrors)
throw createError({ statusCode: 400, statusMessage: options.summary });
const filterPlatforms = options.platform
?.split(",")
.map(parsePlatform)
.filter((e) => e !== undefined);
/**
* Generic filters
*/
@@ -46,23 +51,27 @@ export default defineEventHandler(async (h3) => {
},
}
: undefined;
const platformFilter = options.platform
? {
const platformFilter = filterPlatforms
? ({
versions: {
some: {
launches: {
some: {
platform: {
in: options.platform
.split(",")
.map(parsePlatform)
.filter((e) => e !== undefined),
in: filterPlatforms,
},
},
},
setups: {
some: {
platform: {
in: filterPlatforms,
},
},
},
},
},
}
} satisfies Prisma.GameWhereInput)
: undefined;
/**
server/api/v1/user/mfa/webauthn/index.delete.ts
+49
View File
@@ -0,0 +1,49 @@
import { aclManager } from "~/server/internal/acls";
import { type } from "arktype";
import { readDropValidatedBody, throwingArktype } from "~/server/arktype";
import prisma from "~/server/internal/db/database";
import { MFAMec } from "~/prisma/client/client";
import type { WebAuthNv1Credentials } from "~/server/internal/auth/webauthn";
const WebAuthnDelete = type({
id: "string",
}).configure(throwingArktype);
export default defineEventHandler(async (h3) => {
const userId = await aclManager.allowUserSuperlevel(h3); // No ACLs only allows session authentication
if (!userId)
throw createError({
statusCode: 403,
message: "Not signed in or superlevelled.",
});
const body = await readDropValidatedBody(h3, WebAuthnDelete);
const webauthnMec = await prisma.linkedMFAMec.findUnique({
where: { userId_mec: { userId, mec: MFAMec.WebAuthn } },
});
if (!webauthnMec)
throw createError({ statusCode: 400, message: "WebAuthn not enabled." });
const credentials =
webauthnMec.credentials as unknown as WebAuthNv1Credentials;
const index = credentials.passkeys.findIndex((v) => v.id === body.id);
credentials.passkeys.splice(index, 1);
// SAFETY: we request the object further up
// eslint-disable-next-line drop/no-prisma-delete
await prisma.linkedMFAMec.update({
where: {
userId_mec: {
userId,
mec: MFAMec.WebAuthn,
},
},
data: {
// This works, I don't know why the types don't line up
// eslint-disable-next-line @typescript-eslint/no-explicit-any
credentials: credentials as any,
},
});
});
server/internal/auth/index.ts
+2 -4
View File
@@ -21,10 +21,6 @@ class AuthManager {
},
};
constructor() {
logger.info("AuthManager initialized");
}
async init() {
for (const [key, init] of Object.entries(this.initFuncs)) {
try {
@@ -42,6 +38,8 @@ class AuthManager {
if (!this.authProviders[AuthMec.OpenID]) {
this.authProviders[AuthMec.Simple] = true;
}
logger.info("AuthManager initialized");
}
getAuthProviders() {
server/internal/clients/ca.ts
+9 -13
View File
@@ -1,5 +1,5 @@
import droplet from "@drop-oss/droplet";
import type { CertificateStore } from "./ca-store";
import { dropletInterface } from "../services/torrential/droplet-interface";
export type CertificateBundle = {
priv: string;
@@ -23,8 +23,7 @@ export class CertificateAuthority {
const root = await store.fetch("ca");
let ca;
if (root === undefined) {
const [cert, priv] = droplet.generateRootCa();
const bundle: CertificateBundle = { priv, cert };
const bundle: CertificateBundle = await dropletInterface.generateRootCa();
await store.store("ca", bundle);
ca = new CertificateAuthority(store, bundle);
} else {
@@ -43,16 +42,13 @@ export class CertificateAuthority {
const caCertificate = await this.certificateStore.fetch("ca");
if (!caCertificate)
throw new Error("Certificate authority not initialised");
const [cert, priv] = droplet.generateClientCertificate(
clientId,
clientName,
caCertificate.cert,
caCertificate.priv,
);
const certBundle: CertificateBundle = {
priv,
cert,
};
const certBundle: CertificateBundle =
await dropletInterface.generateClientCert(
clientId,
clientName,
caCertificate,
);
return certBundle;
}
server/internal/clients/event-handler.ts
+14 -38
View File
@@ -1,8 +1,8 @@
import type { ClientModel, UserModel } from "~/prisma/client/models";
import type { EventHandlerRequest, H3Event } from "h3";
import droplet from "@drop-oss/droplet";
import prisma from "../db/database";
import { useCertificateAuthority } from "~/server/plugins/ca";
import jwt from "jsonwebtoken";
export type EventHandlerFunction<T> = (
h3: H3Event<EventHandlerRequest>,
@@ -15,7 +15,8 @@ type ClientUtils = {
fetchUser: () => Promise<UserModel>;
};
const NONCE_LENIENCE = 30_000;
// I forgot how to spell leniancne
const JWT_TIME_WIGGLE = 30_000;
export function defineClientEventHandler<T>(handler: EventHandlerFunction<T>) {
return defineEventHandler(async (h3) => {
@@ -25,39 +26,11 @@ export function defineClientEventHandler<T>(handler: EventHandlerFunction<T>) {
let clientId: string;
switch (method) {
case "Debug": {
if (!import.meta.dev) throw createError({ statusCode: 403 });
const client = await prisma.client.findFirst({ select: { id: true } });
if (!client)
throw createError({
statusCode: 400,
statusMessage: "No clients created.",
});
clientId = client.id;
break;
}
case "Nonce": {
case "JWT": {
clientId = parts[0];
const nonce = parts[1];
const signature = parts[2];
const jwtToken = parts[1];
if (!clientId || !nonce || !signature)
throw createError({ statusCode: 403 });
const nonceTime = parseInt(nonce);
const current = Date.now();
if (
// If it "will be generated" in thirty seconds
nonceTime > current + NONCE_LENIENCE ||
// Or more than thirty seconds ago
nonceTime < current - NONCE_LENIENCE
) {
// We reject the request
throw createError({
statusCode: 403,
statusMessage: "Nonce expired",
});
}
if (!clientId || !jwtToken) throw createError({ statusCode: 403 });
const certificateAuthority = useCertificateAuthority();
const certBundle =
@@ -66,21 +39,24 @@ export function defineClientEventHandler<T>(handler: EventHandlerFunction<T>) {
if (!certBundle)
throw createError({
statusCode: 403,
statusMessage: "Invalid client ID",
message: "Invalid client ID",
});
const valid = droplet.verifyNonce(certBundle.cert, nonce, signature);
const valid = jwt.verify(jwtToken, certBundle.cert, {
clockTolerance: JWT_TIME_WIGGLE,
// algorithms: ["ES384"],
});
if (!valid)
throw createError({
statusCode: 403,
statusMessage: "Invalid nonce signature.",
message: "Invalid nonce signature.",
});
break;
}
default: {
throw createError({
statusCode: 403,
statusMessage: "No authentication",
message: "No authentication",
});
}
}
@@ -88,7 +64,7 @@ export function defineClientEventHandler<T>(handler: EventHandlerFunction<T>) {
if (clientId === undefined)
throw createError({
statusCode: 500,
statusMessage: "Failed to execute authentication pipeline.",
message: "Failed to execute authentication pipeline.",
});
async function fetchClient() {
server/internal/gamesize/index.ts
+91 -203
View File
@@ -1,228 +1,116 @@
import cacheHandler from "../cache";
import prisma from "../db/database";
import { sum } from "../../../utils/array";
import type { Game, GameVersion } from "~/prisma/client/client";
import { castManifest } from "../library/manifest";
import { createDownloadManifestDetails } from "../library/manifest";
import { castManifest } from "../library/manifest/utils";
export type GameSize = {
gameName: string;
size: number;
gameId: string;
export type GameVersionSize = {
versionId: string;
installSize: number;
downloadSize: number;
};
export type VersionSize = GameSize & {
latest: boolean;
};
type VersionsSizes = {
[versionName: string]: VersionSize;
};
type GameVersionsSize = {
[gameId: string]: VersionsSizes;
export type GameSizeBreakdown = {
diskSize: number;
versions: Array<GameVersionSize & { diskSize: number; name: string }>;
};
class GameSizeManager {
private gameVersionsSizesCache =
cacheHandler.createCache<GameVersionsSize>("gameVersionsSizes");
// All versions sizes combined
private gameSizesCache = cacheHandler.createCache<GameSize>("gameSizes");
cacheHandler.createCache<GameVersionSize>("versionSizes");
private gameBreakdownCache =
cacheHandler.createCache<GameSizeBreakdown>("gameBreakdown");
private async clearGameVersionsSizesCache() {
(await this.gameVersionsSizesCache.getKeys()).map((key) =>
this.gameVersionsSizesCache.remove(key),
);
}
private async clearGameSizesCache() {
(await this.gameSizesCache.getKeys()).map((key) =>
this.gameSizesCache.remove(key),
);
}
// All versions of a game combined
async getCombinedGameSize(gameId: string) {
const versions = await prisma.gameVersion.findMany({
where: { gameId },
});
const sizes = await Promise.all(
versions.map((version) => castManifest(version.dropletManifest).size),
);
return sum(sizes);
}
async getGameVersionSize(
gameId: string,
versionId?: string,
): Promise<number | null> {
if (!versionId) {
const version = await prisma.gameVersion.findFirst({
where: { gameId },
orderBy: {
versionIndex: "desc",
},
});
if (!version) {
return null;
}
versionId = version.versionId;
/***
* Gets the size of the game to the user:
* - installSize: size on disk after install
* - downloadSize: how many bytes are downloaded (but not necessarily stored)
*/
async getVersionSize(versionId: string): Promise<GameVersionSize | null> {
if (await this.gameVersionsSizesCache.has(versionId))
return await this.gameVersionsSizesCache.get(versionId);
try {
const { downloadSize, installSize } =
await createDownloadManifestDetails(versionId);
const result = {
downloadSize,
installSize,
versionId,
} satisfies GameVersionSize;
await this.gameVersionsSizesCache.set(versionId, result);
return result;
} catch {
return null;
}
const { dropletManifest } = (await prisma.gameVersion.findUnique({
where: { versionId },
}))!;
return castManifest(dropletManifest).size;
}
private async isLatestVersion(
gameVersions: GameVersion[],
version: GameVersion,
): Promise<boolean> {
return gameVersions.length > 0
? gameVersions[0].versionId === version.versionId
: false;
}
async getBiggestGamesLatestVersion(top: number): Promise<VersionSize[]> {
const gameIds = await this.gameVersionsSizesCache.getKeys();
const latestGames = await Promise.all(
gameIds.map(async (gameId) => {
const versionsSizes = await this.gameVersionsSizesCache.get(gameId);
if (!versionsSizes) {
return null;
}
const latestVersionName = Object.keys(versionsSizes).find(
(versionName) => versionsSizes[versionName].latest,
);
if (!latestVersionName) {
return null;
}
return versionsSizes[latestVersionName] || null;
}),
);
return latestGames
.filter((game) => game !== null)
.sort((gameA, gameB) => gameB.size - gameA.size)
.slice(0, top);
}
async isGameVersionsSizesCacheEmpty() {
return (await this.gameVersionsSizesCache.getKeys()).length === 0;
}
async isGameSizesCacheEmpty() {
return (await this.gameSizesCache.getKeys()).length === 0;
}
async cacheAllCombinedGames() {
await this.clearGameSizesCache();
const games = await prisma.game.findMany({ include: { versions: true } });
await Promise.all(games.map((game) => this.cacheCombinedGame(game)));
}
async cacheCombinedGame(game: Game) {
const size = await this.getCombinedGameSize(game.id);
if (!size) {
this.gameSizesCache.remove(game.id);
return;
}
const gameSize = {
size,
gameName: game.mName,
gameId: game.id,
};
await this.gameSizesCache.set(game.id, gameSize);
}
async cacheAllGameVersions() {
await this.clearGameVersionsSizesCache();
const games = await prisma.game.findMany({
include: {
versions: {
orderBy: {
versionIndex: "desc",
},
take: 1,
},
/***
* Get the size of the game on disk
*/
async getVersionDiskSize(versionId: string): Promise<number | null> {
const version = await prisma.gameVersion.findUnique({
where: {
versionId,
},
select: {
dropletManifest: true,
},
});
await Promise.all(games.map((game) => this.cacheGameVersion(game)));
if (!version) return null;
return castManifest(version.dropletManifest).size;
}
async cacheGameVersion(
game: Game & { versions: GameVersion[] },
versionId?: string,
) {
const cacheVersion = async (version: GameVersion) => {
const size = await this.getGameVersionSize(game.id, version.versionId);
if (!version.versionId || !size) {
return;
}
const versionsSizes = {
[version.versionId]: {
size,
gameName: game.mName,
gameId: game.id,
latest: await this.isLatestVersion(game.versions, version),
},
};
const allVersionsSizes =
(await this.gameVersionsSizesCache.get(game.id)) || {};
await this.gameVersionsSizesCache.set(game.id, {
...allVersionsSizes,
...versionsSizes,
});
};
if (versionId) {
const version = await prisma.gameVersion.findFirst({
where: { gameId: game.id, versionId },
});
if (!version) {
return;
}
cacheVersion(version);
return;
}
if ("versions" in game) {
await Promise.all(game.versions.map(cacheVersion));
}
}
async getBiggestGamesAllVersions(top: number): Promise<GameSize[]> {
const gameIds = await this.gameSizesCache.getKeys();
const allGames = await Promise.all(
gameIds.map(async (gameId) => await this.gameSizesCache.get(gameId)),
/**
* Calculate the total disk usage of a game
* @param gameId Game ID to calculate
* @returns Total **disk** size of the game
*/
async getGameDiskSize(gameId: string): Promise<number> {
const versions = await prisma.gameVersion.findMany({
where: { gameId },
select: {
versionId: true,
},
});
const sizes = await Promise.all(
versions.map((version) => this.getVersionDiskSize(version.versionId)),
);
return allGames
.filter((game) => game !== null)
.sort((gameA, gameB) => gameB.size - gameA.size)
.slice(0, top);
return sum(sizes.filter((v) => v !== null));
}
async deleteGameVersion(gameId: string, version: string) {
const game = await prisma.game.findFirst({ where: { id: gameId } });
if (game) {
await this.cacheCombinedGame(game);
}
const versionsSizes = await this.gameVersionsSizesCache.get(gameId);
if (!versionsSizes) {
return;
}
// Remove the version from the VersionsSizes object
const { [version]: _, ...updatedVersionsSizes } = versionsSizes;
await this.gameVersionsSizesCache.set(gameId, updatedVersionsSizes);
}
async getGameBreakdown(gameId: string): Promise<GameSizeBreakdown | null> {
const versions = await prisma.gameVersion.findMany({
where: { gameId },
orderBy: { versionIndex: "desc" },
select: { versionId: true, displayName: true, versionPath: true },
});
if (!versions) return null;
async deleteGame(gameId: string) {
this.gameSizesCache.remove(gameId);
this.gameVersionsSizesCache.remove(gameId);
const breakdownKey = `${gameId} ${versions.map((v) => v.versionId).join(" ")}`;
if (await this.gameBreakdownCache.has(breakdownKey))
return (await this.gameBreakdownCache.get(breakdownKey))!;
let diskSize = 0;
const versionInformation = [];
for (const version of versions) {
const size = (await this.getVersionSize(version.versionId))!;
const vDiskSize = (await this.getVersionDiskSize(version.versionId))!;
diskSize += vDiskSize;
versionInformation.push({
...size,
diskSize: vDiskSize,
name: (version.displayName ?? version.versionPath)!,
});
}
const result = {
diskSize,
versions: versionInformation,
};
await this.gameBreakdownCache.set(breakdownKey, result);
return result;
}
}
export const manager = new GameSizeManager();
export default manager;
export const gameSizeManager = new GameSizeManager();
export default gameSizeManager;
server/internal/library/index.ts
+9 -68
View File
@@ -16,10 +16,9 @@ import type { GameModel } from "~/prisma/client/models";
import { createHash } from "node:crypto";
import type { WorkingLibrarySource } from "~/server/api/v1/admin/library/sources/index.get";
import gameSizeManager from "~/server/internal/gamesize";
import { TORRENTIAL_SERVICE } from "../services/services/torrential";
import type { ImportVersion } from "~/server/api/v1/admin/import/version/index.post";
import { GameType, type Platform } from "~/prisma/client/enums";
import { castManifest } from "./manifest";
import { castManifest } from "./manifest/utils";
export function createGameImportTaskId(libraryId: string, libraryPath: string) {
return createHash("md5")
@@ -417,12 +416,10 @@ class LibraryManager {
manifest = await library.generateDropletManifest(
game.libraryPath,
versionPath,
(err, value) => {
if (err) throw err;
(value) => {
progress(value * 0.9);
},
(err, value) => {
if (err) throw err;
(value) => {
logger.info(value);
},
);
@@ -500,13 +497,12 @@ class LibraryManager {
acls: ["system:import:version:read"],
});
await libraryManager.cacheCombinedGameSize(gameId);
await libraryManager.cacheGameVersionSize(gameId, newVersion.versionId);
await TORRENTIAL_SERVICE.utils().invalidate(
gameId,
newVersion.versionId,
);
// Ensure cache is filled (also pre-caches the manifest)
try {
await gameSizeManager.getVersionSize(newVersion.versionId);
} catch (e) {
logger.warn(`Failed to pre-cache game size and manifest: ${e}`);
}
if (version.type === "depot") {
// SAFETY: we can only reach this if the type is depot and identifier is valid
@@ -533,18 +529,6 @@ class LibraryManager {
return await library.peekFile(game, version, filename);
}
async readFile(
libraryId: string,
game: string,
version: string,
filename: string,
options?: { start?: number; end?: number },
) {
const library = this.libraries.get(libraryId);
if (!library) return undefined;
return await library.readFile(game, version, filename, options);
}
async deleteGameVersion(gameId: string, version: string) {
await prisma.gameVersion.deleteMany({
where: {
@@ -552,8 +536,6 @@ class LibraryManager {
versionId: version,
},
});
await gameSizeManager.deleteGameVersion(gameId, version);
}
async deleteGame(gameId: string) {
@@ -562,7 +544,6 @@ class LibraryManager {
id: gameId,
},
});
await gameSizeManager.deleteGame(gameId);
// Delete all game versions that depended on this game
await prisma.gameVersion.deleteMany({
where: {
@@ -578,46 +559,6 @@ class LibraryManager {
},
});
}
async getGameVersionSize(
gameId: string,
versionName?: string,
): Promise<number | null> {
return gameSizeManager.getGameVersionSize(gameId, versionName);
}
async getBiggestGamesCombinedVersions(top: number) {
if (await gameSizeManager.isGameSizesCacheEmpty()) {
await gameSizeManager.cacheAllCombinedGames();
}
return gameSizeManager.getBiggestGamesAllVersions(top);
}
async getBiggestGamesLatestVersions(top: number) {
if (await gameSizeManager.isGameVersionsSizesCacheEmpty()) {
await gameSizeManager.cacheAllGameVersions();
}
return gameSizeManager.getBiggestGamesLatestVersion(top);
}
async cacheCombinedGameSize(gameId: string) {
const game = await prisma.game.findFirst({ where: { id: gameId } });
if (!game) {
return;
}
await gameSizeManager.cacheCombinedGame(game);
}
async cacheGameVersionSize(gameId: string, versionId: string) {
const game = await prisma.game.findFirst({
where: { id: gameId },
include: { versions: true },
});
if (!game) {
return;
}
await gameSizeManager.cacheGameVersion(game, versionId);
}
}
export const libraryManager = new LibraryManager();
server/internal/library/manifest/index.ts
+46 -6
View File
@@ -1,14 +1,27 @@
import cacheHandler from "../../cache";
import prisma from "../../db/database";
import { castManifest, type DropletManifest } from "../manifest";
import { castManifest, type DropletManifest } from "./utils";
export type DownloadManifestDetails = {
/***
* Version ID to manifest
*/
manifests: { [key: string]: DropletManifest };
/***
* File name to version ID
*/
fileList: { [key: string]: string };
/// Size on disk after download
installSize: number;
/// Size of download
downloadSize: number;
};
function convertMap<T>(map: Map<string, T>): { [key: string]: T } {
return Object.fromEntries(map.entries().toArray());
}
const manifestCache =
cacheHandler.createCache<DownloadManifestDetails>("manifestCache");
/**
*
@@ -17,7 +30,10 @@ function convertMap<T>(map: Map<string, T>): { [key: string]: T } {
*/
export async function createDownloadManifestDetails(
versionId: string,
refresh = false,
): Promise<DownloadManifestDetails> {
if ((await manifestCache.has(versionId)) && !refresh)
return (await manifestCache.get(versionId))!;
const mainVersion = await prisma.gameVersion.findUnique({
where: { versionId },
select: {
@@ -35,7 +51,7 @@ export async function createDownloadManifestDetails(
const collectedVersions = [];
let versionIndex = mainVersion.versionIndex;
while (true) {
while (mainVersion.delta) {
const nextVersion = await prisma.gameVersion.findFirst({
where: { gameId: mainVersion.gameId, versionIndex: { lt: versionIndex } },
orderBy: {
@@ -75,6 +91,9 @@ export async function createDownloadManifestDetails(
}
}
let installSize = 0;
let downloadSize = 0;
// Now that we have our file list, filter the manifests
const manifests = new Map<string, DropletManifest>();
for (const version of versionOrder) {
@@ -86,9 +105,22 @@ export async function createDownloadManifestDetails(
const fileNames = Object.fromEntries(files);
const manifest = castManifest(version.dropletManifest);
const filteredChunks = Object.fromEntries(
Object.entries(manifest.chunks).filter(([, chunkData]) =>
chunkData.files.some((fileEntry) => !!fileNames[fileEntry.filename]),
),
Object.entries(manifest.chunks).filter(([, chunkData]) => {
let flag = false;
chunkData.files.forEach((fileEntry) => {
if (fileNames[fileEntry.filename]) {
flag = true;
installSize += fileEntry.length;
}
});
// If we have to download this chunk, add it's length
if (flag) {
downloadSize += chunkData.files
.map((v) => v.length)
.reduce((a, b) => a + b, 0);
}
return flag;
}),
);
manifests.set(version.versionId, {
...manifest,
@@ -96,5 +128,13 @@ export async function createDownloadManifestDetails(
});
}
return { fileList: convertMap(fileList), manifests: convertMap(manifests) };
const result = {
fileList: convertMap(fileList),
manifests: convertMap(manifests),
installSize,
downloadSize,
};
await manifestCache.set(versionId, result);
return result;
}
server/internal/library/manifest.ts → server/internal/library/manifest/utils.ts
View File
server/internal/library/provider.ts
+2 -9
View File
@@ -44,8 +44,8 @@ export abstract class LibraryProvider<CFG> {
abstract generateDropletManifest(
game: string,
version: string,
progress: (err: Error | null, v: number) => void,
log: (err: Error | null, v: string) => void,
progress: (v: number) => void,
log: (v: string) => void,
): Promise<string>;
abstract peekFile(
@@ -54,13 +54,6 @@ export abstract class LibraryProvider<CFG> {
filename: string,
): Promise<{ size: number } | undefined>;
abstract readFile(
game: string,
version: string,
filename: string,
options?: { start?: number; end?: number },
): Promise<ReadableStream | undefined>;
abstract fsStats(): { freeSpace: number; totalSpace: number } | undefined;
}
server/internal/library/providers/filesystem.ts
+21 -34
View File
@@ -7,13 +7,8 @@ import {
import { LibraryBackend } from "~/prisma/client/enums";
import fs from "fs";
import path from "path";
import droplet, {
hasBackendForPath,
listFiles,
peekFile,
readFile,
} from "@drop-oss/droplet";
import { fsStats } from "~/server/internal/utils/files";
import { dropletInterface } from "../../services/torrential/droplet-interface";
export const FilesystemProviderConfig = type({
baseDir: "string",
@@ -64,57 +59,49 @@ export class FilesystemProvider
const gameDir = path.join(this.config.baseDir, game);
if (!fs.existsSync(gameDir)) throw new GameNotFoundError();
const versionDirs = fs.readdirSync(gameDir);
const validVersionDirs = versionDirs.filter((e) => {
if (ignoredVersions && ignoredVersions.includes(e)) return false;
const fullDir = path.join(this.config.baseDir, game, e);
return hasBackendForPath(fullDir);
});
const validVersionDirs = [];
for (const versionDir of versionDirs) {
if (ignoredVersions && ignoredVersions.includes(versionDir)) continue;
const fullDir = path.join(this.config.baseDir, game, versionDir);
const valid = await dropletInterface.hasBackend(fullDir);
if (!valid) continue;
validVersionDirs.push(versionDir);
}
return validVersionDirs;
}
async versionReaddir(game: string, version: string): Promise<string[]> {
const versionDir = path.join(this.config.baseDir, game, version);
if (!fs.existsSync(versionDir)) throw new VersionNotFoundError();
return await listFiles(versionDir);
return await dropletInterface.listFiles(versionDir);
}
async generateDropletManifest(
game: string,
version: string,
progress: (err: Error | null, v: number) => void,
log: (err: Error | null, v: string) => void,
progress: (v: number) => void,
log: (v: string) => void,
): Promise<string> {
const versionDir = path.join(this.config.baseDir, game, version);
if (!fs.existsSync(versionDir)) throw new VersionNotFoundError();
const manifest = await droplet.generateManifest(versionDir, progress, log);
const manifest = await dropletInterface.generateDropletManifest(
versionDir,
progress,
log,
);
return manifest;
}
async peekFile(game: string, version: string, filename: string) {
const filepath = path.join(this.config.baseDir, game, version);
if (!fs.existsSync(filepath)) return undefined;
const stat = await peekFile(filepath, filename);
const stat = await dropletInterface.peekFile(filepath, filename);
return { size: Number(stat) };
}
async readFile(
game: string,
version: string,
filename: string,
options?: { start?: number; end?: number },
) {
const filepath = path.join(this.config.baseDir, game, version);
if (!fs.existsSync(filepath)) return undefined;
const stream = await readFile(
filepath,
filename,
options?.start ? BigInt(options.start) : undefined,
options?.end ? BigInt(options.end) : undefined,
);
return stream;
}
fsStats() {
return fsStats(this.config.baseDir);
}
server/internal/library/providers/flat.ts
+20 -33
View File
@@ -4,13 +4,8 @@ import { VersionNotFoundError } from "../provider";
import { LibraryBackend } from "~/prisma/client/enums";
import fs from "fs";
import path from "path";
import droplet, {
hasBackendForPath,
listFiles,
peekFile,
readFile,
} from "@drop-oss/droplet";
import { fsStats } from "~/server/internal/utils/files";
import { dropletInterface } from "../../services/torrential/droplet-interface";
export const FlatFilesystemProviderConfig = type({
baseDir: "string",
@@ -50,10 +45,15 @@ export class FlatFilesystemProvider
*/
async listGames() {
const versionDirs = fs.readdirSync(this.config.baseDir);
const validVersionDirs = versionDirs.filter((e) => {
const fullDir = path.join(this.config.baseDir, e);
return hasBackendForPath(fullDir);
});
const validVersionDirs = [];
for (const versionDir of versionDirs) {
const fullDir = path.join(this.config.baseDir, versionDir);
const valid = await dropletInterface.hasBackend(fullDir);
if (!valid) continue;
validVersionDirs.push(versionDir);
}
return validVersionDirs;
}
@@ -69,44 +69,31 @@ export class FlatFilesystemProvider
async versionReaddir(game: string, _version: string) {
const versionDir = path.join(this.config.baseDir, game);
if (!fs.existsSync(versionDir)) throw new VersionNotFoundError();
return await listFiles(versionDir);
return await dropletInterface.listFiles(versionDir);
}
async generateDropletManifest(
game: string,
_version: string,
progress: (err: Error | null, v: number) => void,
log: (err: Error | null, v: string) => void,
progress: (v: number) => void,
log: (v: string) => void,
) {
const versionDir = path.join(this.config.baseDir, game);
if (!fs.existsSync(versionDir)) throw new VersionNotFoundError();
const manifest = await droplet.generateManifest(versionDir, progress, log);
const manifest = await dropletInterface.generateDropletManifest(
versionDir,
progress,
log,
);
return manifest;
}
async peekFile(game: string, _version: string, filename: string) {
const filepath = path.join(this.config.baseDir, game);
if (!fs.existsSync(filepath)) return undefined;
const stat = await peekFile(filepath, filename);
const stat = await dropletInterface.peekFile(filepath, filename);
return { size: Number(stat) };
}
async readFile(
game: string,
_version: string,
filename: string,
options?: { start?: number; end?: number },
) {
const filepath = path.join(this.config.baseDir, game);
if (!fs.existsSync(filepath)) return undefined;
const stream = await readFile(
filepath,
filename,
options?.start ? BigInt(options.start) : undefined,
options?.end ? BigInt(options.end) : undefined,
);
if (!stream) return undefined;
return stream;
}
fsStats() {
return fsStats(this.config.baseDir);
server/internal/services/index.ts
+5 -1
View File
@@ -38,7 +38,7 @@ export class Service<T> {
private setup: Setup | undefined;
private healthcheck: Healthcheck | undefined;
private logger: Logger<never>;
logger: Logger<never>;
private currentProcess: ChildProcess | undefined;
@@ -90,6 +90,7 @@ export class Service<T> {
if (!process.env[disableEnv]) {
const serviceProcess = this.executor();
this.logger.info("service launched");
serviceProcess.on("close", async (code, signal) => {
serviceProcess.kill();
this.currentProcess = undefined;
@@ -99,12 +100,15 @@ export class Service<T> {
await new Promise((r) => setTimeout(r, 5000));
if (this.spun) this.launch();
});
serviceProcess.stdout?.on("data", (data) =>
this.logger.info(data.toString().trim()),
);
serviceProcess.stderr?.on("data", (data) =>
this.logger.error(data.toString().trim()),
);
this.currentProcess = serviceProcess;
}
server/internal/services/services/torrential.ts
-78
View File
@@ -1,78 +0,0 @@
import { spawn } from "child_process";
import { Service } from "..";
import fs from "fs";
import prisma from "../../db/database";
import { logger } from "../../logging";
import { systemConfig } from "../../config/sys-conf";
const INTERNAL_DEPOT_URL = new URL(
process.env.INTERNAL_DEPOT_URL ?? "http://localhost:5000",
);
export const TORRENTIAL_SERVICE = new Service(
"torrential",
() => {
const localDir = fs.readdirSync(".");
if ("torrential" in localDir) {
const stat = fs.statSync("./torrential");
if (stat.isDirectory()) {
// in dev and we have the submodule
logger.info(
"torrential detected in development mode - building from source",
);
return spawn(
"cargo run --manifest-path ./torrential/Cargo.toml",
[],
{},
);
} else {
// binary
return spawn("./torrential", [], {});
}
}
const envPath = process.env.TORRENTIAL_PATH;
if (envPath) return spawn(envPath, [], {});
return spawn("torrential", [], {});
},
async () => {
const externalUrl = systemConfig.getExternalUrl();
const depot = await prisma.depot.upsert({
where: {
id: "torrential",
},
update: {
endpoint: `${externalUrl}/api/v1/depot`,
},
create: {
id: "torrential",
endpoint: `${externalUrl}/api/v1/depot`,
},
});
await $fetch(`${INTERNAL_DEPOT_URL.toString()}key`, {
method: "POST",
body: { key: depot.key },
});
return true;
},
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
async () => await $fetch(`${INTERNAL_DEPOT_URL.toString()}healthcheck`),
{
async invalidate(gameId: string, versionId: string) {
try {
await $fetch(`${INTERNAL_DEPOT_URL.toString()}invalidate`, {
method: "POST",
body: {
game: gameId,
version: versionId,
},
});
} catch (e) {
logger.warn("invalidate torrential cache failed with error: " + e);
}
},
},
);
server/internal/services/torrential/README.md
+27
View File
@@ -0,0 +1,27 @@
# torrential service
The role of torrential has expanded recently to be the source of ALL Rust/native execution within Drop, to avoid using the buggy napi.rs `droplet` package.
It communicates over `127.0.0.1:33148`, which the service connects to and stores the socket handle to.
## message format
Each message is prefixed with an 8 byte little-endian unsigned integer that dictates the length of the message. Then, they are wrapped in the respective DropBound or TorrentialBound wrappers, which contain the type and data fields, which dictate which sub-message they are deserialized into.
## query processors
**Note: "Query" is the old name for a DropBound message**
The service allows you to configure a series of query processors that match based on type and recieve the raw message to deserialize themselves. They can optionally return a response message, which automatically gets returned and wrapped.
## message ids
All messages in the pipe have a message ID which dictates which "request" they're for. Queries and responses (DropBound and TorrentialBound) carry the same message ID if they are related.
## old `/api/v1/admin/depot/torrential/*` routes
They've been turned into query and response messages as described above.
# torrential service internals
We use a read buffer to queue up enough bytes that we can deserialize the entire message at once. When a chunk comes in, we append it to the current readbuf, and then check if we have enough bytes to assemble the length header and it's associated packet. If we do, we deserialize, cut off the bytes, and fire off all the necessary handlers for that packet.
server/internal/services/torrential/droplet-interface.ts
+361
View File
@@ -0,0 +1,361 @@
import type { Message } from "@bufbuild/protobuf";
import { create, fromBinary } from "@bufbuild/protobuf";
import {
ClientCertQuerySchema,
ClientCertResponseSchema,
GenerateManifestSchema,
HasBackendQuerySchema,
HasBackendResponseSchema,
ListFilesQuerySchema,
ListFilesResponseSchema,
ManifestCompleteSchema,
ManifestLogSchema,
ManifestProgressSchema,
PeekFileQuerySchema,
PeekFileResponseSchema,
RootCertQuerySchema,
RootCertResponseSchema,
RpcErrorSchema,
} from "../../proto/torrential/proto/droplet_pb";
import type { QueryProcessor } from ".";
import TORRENTIAL_SERVICE from ".";
import type { DropBound } from "../../proto/torrential/proto/core_pb";
import {
DropBoundType,
TorrentialBoundType,
} from "../../proto/torrential/proto/core_pb";
import { logger } from "../../logging";
import type { CertificateBundle } from "../../clients/ca";
import type { GenMessage } from "@bufbuild/protobuf/codegenv2";
interface BaseCallbacks<T> {
resolve: (value: T) => void;
reject: (err: string) => void;
}
type ManifestGenerationCallbacks = BaseCallbacks<string> & {
progress: (v: number) => void;
log: (v: string) => void;
type: "manifest";
};
type CaGenerationCallback = BaseCallbacks<CertificateBundle> & {
type: "certificate";
};
type HasBackendCallback = BaseCallbacks<boolean> & {
type: "has_backend";
};
type ListFilesCallback = BaseCallbacks<string[]> & {
type: "list_files";
};
type PeekFileCallback = BaseCallbacks<number> & {
type: "peek_file";
};
type DropletFunctionCallbacks =
| ManifestGenerationCallbacks
| CaGenerationCallback
| HasBackendCallback
| ListFilesCallback
| PeekFileCallback;
class DropletInterfaceManager {
private callbacks: Map<string, DropletFunctionCallbacks> = new Map();
private queryProcessors: QueryProcessor<
DropBoundType,
TorrentialBoundType,
Message
>[];
constructor() {
// This handler is special, it's a global error handler
const errorProcessor = this.defineDropletCallbackProcessor({
queryType: DropBoundType.RPC_ERROR,
run: async (message, callbacks) => {
const messageData = fromBinary(RpcErrorSchema, message.data);
callbacks.reject(messageData.error);
this.callbacks.delete(message.messageId);
},
});
// Other than the error handler, each "_COMPLETE" handler is responsible
// for resolving the promise, and cleaning themselves up (removing from map)
const manifestCompleteProcessor = this.defineDropletCallbackProcessor({
queryType: DropBoundType.MANIFEST_COMPLETE,
callbackType: "manifest",
run: async (message, callbacks) => {
const messageData = fromBinary(ManifestCompleteSchema, message.data);
callbacks.resolve(messageData.manifest);
this.callbacks.delete(message.messageId);
},
});
const manifestLogProcessor = this.defineDropletCallbackProcessor({
queryType: DropBoundType.MANIFEST_LOG,
callbackType: "manifest",
run: async (message, callbacks) => {
const messageData = fromBinary(ManifestLogSchema, message.data);
callbacks.log(messageData.logLine);
},
});
const manifestProgressProcessor = this.defineDropletCallbackProcessor({
queryType: DropBoundType.MANIFEST_PROGRESS,
callbackType: "manifest",
run: async (message, callbacks) => {
const messageData = fromBinary(ManifestProgressSchema, message.data);
callbacks.progress(messageData.progress);
},
});
const rootCaProcessor = this.defineDropletCallbackProcessor({
queryType: DropBoundType.ROOT_CA_COMPLETE,
callbackType: "certificate",
run: async (message, callbacks) => {
const messageData = fromBinary(RootCertResponseSchema, message.data);
callbacks.resolve({
priv: messageData.priv,
cert: messageData.cert,
} satisfies CertificateBundle);
this.callbacks.delete(message.messageId);
},
});
const clientCertProcessor = this.defineDropletCallbackProcessor({
queryType: DropBoundType.CLIENT_CERT_COMPLETE,
callbackType: "certificate",
run: async (message, callbacks) => {
const messageData = fromBinary(ClientCertResponseSchema, message.data);
callbacks.resolve({
cert: messageData.cert,
priv: messageData.priv,
});
this.callbacks.delete(message.messageId);
},
});
const hasBackendProcessor = this.defineDropletCallbackProcessor({
queryType: DropBoundType.HAS_BACKEND_COMPLETE,
callbackType: "has_backend",
run: async (message, callbacks) => {
const messageData = fromBinary(HasBackendResponseSchema, message.data);
callbacks.resolve(messageData.result);
this.callbacks.delete(message.messageId);
},
});
const listFilesProcessor = this.defineDropletCallbackProcessor({
queryType: DropBoundType.LIST_FILES_COMPLETE,
callbackType: "list_files",
run: async (message, callbacks) => {
const messageData = fromBinary(ListFilesResponseSchema, message.data);
callbacks.resolve(messageData.files);
this.callbacks.delete(message.messageId);
},
});
const peekFileProcessor = this.defineDropletCallbackProcessor({
queryType: DropBoundType.PEEK_FILE_COMPLETE,
callbackType: "peek_file",
run: async (message, callbacks) => {
const messageData = fromBinary(PeekFileResponseSchema, message.data);
callbacks.resolve(Number(messageData.size));
this.callbacks.delete(message.messageId);
},
});
// All query processors go into the array to get added
this.queryProcessors = [
errorProcessor,
manifestCompleteProcessor,
manifestLogProcessor,
manifestProgressProcessor,
rootCaProcessor,
clientCertProcessor,
hasBackendProcessor,
listFilesProcessor,
peekFileProcessor,
];
for (const processor of this.queryProcessors) {
TORRENTIAL_SERVICE.registerProcessor(processor);
}
}
/**
* Defines a handler to consume an incoming message
* from torrential
*
* Passes in the query type (DropBoundType) and callback type,
* to make sure we respond to right callback,
* and give us proper typing when it comes to the callbacks (resolve, specifically)
*
* Returns a query processor that can be registered with the service
*/
private defineDropletCallbackProcessor<
T extends DropBoundType,
K extends TorrentialBoundType,
V extends Message,
C extends DropletFunctionCallbacks,
CT extends C["type"],
>(opts: {
queryType: T;
callbackType?: CT;
run: (
query: DropBound,
callbacks: Extract<C, { type: CT }>,
) => Promise<void>;
}) {
return {
queryType: opts.queryType,
run: async (message) => {
const callbacks = this.callbacks.get(message.messageId);
if (!callbacks) {
logger.warn(
`got a droplet message with old message id: ${message.type}, ${message.messageId}`,
);
return undefined;
}
if (opts.callbackType && callbacks.type !== opts.callbackType)
return undefined;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
await opts.run(message, callbacks as any);
return undefined;
},
} satisfies QueryProcessor<T, K, V>;
}
getProcessors() {
return this.queryProcessors;
}
/**
* Sets up message ID,
* sends request to torrential,
* and sets up callbacks
*/
private async createDropletFunction<
M extends Message,
K extends DropletFunctionCallbacks,
KT extends K["type"],
>(
message: M,
schema: GenMessage<M>,
messageType: TorrentialBoundType,
callbackType: KT,
): Promise<Parameters<Extract<K, { type: KT }>["resolve"]>[0]> {
const messageId = crypto.randomUUID();
await TORRENTIAL_SERVICE.writeMessage(messageId, {
type: messageType,
schema: schema,
data: message,
});
return await new Promise((resolve, reject) => {
this.callbacks.set(messageId, {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
type: callbackType as any,
resolve,
reject,
});
});
}
async generateDropletManifest(
versionDir: string,
progress: (v: number) => void,
log: (v: string) => void,
) {
const messageId = crypto.randomUUID();
const manifestGenerationRequest = create(GenerateManifestSchema, {
versionDir,
});
await TORRENTIAL_SERVICE.writeMessage(messageId, {
type: TorrentialBoundType.GENERATE_MANIFEST,
schema: GenerateManifestSchema,
data: manifestGenerationRequest,
});
return await new Promise<string>((resolve, reject) => {
this.callbacks.set(messageId, {
resolve,
reject,
progress,
log,
type: "manifest",
});
});
}
async generateRootCa() {
return await this.createDropletFunction(
create(RootCertQuerySchema, {}),
RootCertQuerySchema,
TorrentialBoundType.GENERATE_ROOT_CA,
"certificate",
);
}
async generateClientCert(
clientId: string,
clientName: string,
rootCa: CertificateBundle,
) {
return await this.createDropletFunction(
create(ClientCertQuerySchema, {
clientId,
clientName,
rootPriv: rootCa.priv,
rootCert: rootCa.cert,
}),
ClientCertQuerySchema,
TorrentialBoundType.GENERATE_CLIENT_CERT,
"certificate",
);
}
async hasBackend(path: string) {
return await this.createDropletFunction(
create(HasBackendQuerySchema, {
path,
}),
HasBackendQuerySchema,
TorrentialBoundType.HAS_BACKEND_QUERY,
"has_backend",
);
}
async listFiles(path: string) {
return await this.createDropletFunction(
create(ListFilesQuerySchema, {
path,
}),
ListFilesQuerySchema,
TorrentialBoundType.LIST_FILES_QUERY,
"list_files",
);
}
async peekFile(path: string, subpath: string) {
return await this.createDropletFunction(
create(PeekFileQuerySchema, {
path: path,
filename: subpath,
}),
PeekFileQuerySchema,
TorrentialBoundType.PEEK_FILE_QUERY,
"peek_file",
);
}
}
export const dropletInterface = new DropletInterfaceManager();
export default dropletInterface;
server/internal/services/torrential/index.ts
+189
View File
@@ -0,0 +1,189 @@
import { spawn } from "child_process";
import { Service } from "..";
import fs from "fs";
import { logger } from "../../logging";
import type { Socket } from "net";
import net from "net";
import { create, toBinary, type Message } from "@bufbuild/protobuf";
import { fromBinary } from "@bufbuild/protobuf";
import { StringValueSchema } from "@bufbuild/protobuf/wkt";
import type { GenMessage } from "@bufbuild/protobuf/codegenv2";
import {
DropBoundSchema,
TorrentialBoundSchema,
TorrentialBoundType,
type DropBound,
type DropBoundType,
} from "../../proto/torrential/proto/core_pb";
/// Processors
import manifestFetchProcessor from "./manifest-fetch";
import serverGamesProcessor from "./server-games";
const INTERNAL_DEPOT_URL = new URL(
process.env.INTERNAL_DEPOT_URL ?? "http://localhost:5000",
);
export interface QueryProcessor<
T extends DropBoundType,
K extends TorrentialBoundType,
V extends Message,
> {
queryType: T;
run: (
query: DropBound,
) => Promise<{ type: K; schema: GenMessage<V>; data: V } | undefined>;
}
export class TorrentialService extends Service<unknown> {
private socket: Socket | undefined;
private readbuf: Buffer<ArrayBufferLike> = Buffer.alloc(0);
private readingQueue = false;
private queryProcessors: Map<
DropBoundType,
QueryProcessor<DropBoundType, TorrentialBoundType, Message>
> = new Map();
constructor() {
super(
"torrential",
() => {
const localDir = fs.readdirSync(".");
if ("torrential" in localDir) {
const stat = fs.statSync("./torrential");
if (stat.isDirectory()) {
// in dev and we have the submodule
logger.info(
"torrential detected in development mode - building from source",
);
return spawn(
"cargo run --manifest-path ./torrential/Cargo.toml",
[],
{},
);
} else {
// binary
return spawn("./torrential", [], {});
}
}
const envPath = process.env.TORRENTIAL_PATH;
if (envPath) return spawn(envPath, [], {});
return spawn("torrential", [], {});
},
async () => {
if (this.socket) return true;
this.socket = net.createConnection({ port: 33148, host: "127.0.0.1" });
await new Promise<void>((r) =>
this.socket!.on("connect", () => {
this.logger.info("connected to torrential socket");
r();
}),
);
this.setupRead();
return true;
},
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
async () => await $fetch(`${INTERNAL_DEPOT_URL.toString()}healthcheck`),
{},
);
this.queryProcessors.set(
manifestFetchProcessor.queryType,
manifestFetchProcessor,
);
this.queryProcessors.set(
serverGamesProcessor.queryType,
serverGamesProcessor,
);
}
registerProcessor(
processor: QueryProcessor<DropBoundType, TorrentialBoundType, Message>,
) {
this.queryProcessors.set(processor.queryType, processor);
}
private setupRead() {
if (!this.socket) return;
this.socket.on("data", (data) => {
this.readbuf = Buffer.concat([this.readbuf, data]);
if (!this.readingQueue) {
this.readingQueue = true;
this.queueRead().finally(() => {
this.readingQueue = false;
});
}
});
}
async writeMessage<T extends Message>(
messageId: string,
value: {
type: TorrentialBoundType;
schema: GenMessage<T>;
data: T;
},
) {
const response = create(TorrentialBoundSchema, {
messageId: messageId,
type: value.type,
data: toBinary(value.schema, value.data),
});
const responseBinary = toBinary(TorrentialBoundSchema, response);
const responseLength = responseBinary.length;
const responseLengthBuf = Buffer.allocUnsafe(8);
responseLengthBuf.writeBigUInt64LE(BigInt(responseLength), 0);
this.socket!.write(responseLengthBuf);
this.socket!.write(responseBinary);
}
private async queueRead() {
if (this.readbuf.length < 8) return;
const sizeBytes = this.readbuf.subarray(0, 8);
const size = sizeBytes.readBigUInt64LE(0);
const end = Number(size + BigInt(8));
if (this.readbuf.length < end) return;
const buffer = this.readbuf.subarray(8, end);
this.readbuf = this.readbuf.subarray(end);
const query = fromBinary(DropBoundSchema, buffer);
const processor = this.queryProcessors.get(query.type);
if (!processor) {
this.logger.warn(`no processor for query type: ${query.type}`);
return;
}
let value;
try {
value = await processor.run(query);
} catch (e) {
this.logger.warn(
`process query for ${query.type} failed with error: ${e}`,
);
value = {
type: TorrentialBoundType.ERROR,
schema: StringValueSchema,
data: create(StringValueSchema, {
value: (e as string).toString(),
}),
};
}
if (value) await this.writeMessage(query.messageId, value);
// Call until we can't
await this.queueRead();
}
}
export const TORRENTIAL_SERVICE = new TorrentialService();
export default TORRENTIAL_SERVICE;
server/internal/services/torrential/manifest-fetch.ts
+89
View File
@@ -0,0 +1,89 @@
import {
VersionQuerySchema,
VersionResponse_LibrarySource_LibraryBackend,
VersionResponse_LibrarySourceSchema,
VersionResponse_Manifest_ChunkData_FileEntrySchema,
VersionResponse_Manifest_ChunkDataSchema,
VersionResponse_ManifestSchema,
VersionResponseSchema,
} from "../../proto/torrential/proto/version_pb";
import { castManifest } from "../../library/manifest/utils";
import { LibraryBackend } from "~/prisma/client/client";
import { create, fromBinary } from "@bufbuild/protobuf";
import prisma from "../../db/database";
import { defineQueryProcessor } from "./utils";
import {
DropBoundType,
TorrentialBoundType,
} from "../../proto/torrential/proto/core_pb";
export default defineQueryProcessor({
queryType: DropBoundType.VERSION_QUERY,
run: async (query) => {
const queryData = fromBinary(VersionQuerySchema, query.data);
const version = await prisma.gameVersion.findUnique({
where: {
versionId: queryData.versionId,
},
select: {
dropletManifest: true,
versionPath: true,
game: {
select: {
library: true,
libraryPath: true,
},
},
},
});
if (!version) throw "Game version not found";
const manifest = castManifest(version.dropletManifest);
const mapEnum = (v: LibraryBackend) => {
switch (v) {
case LibraryBackend.Filesystem:
return VersionResponse_LibrarySource_LibraryBackend.FILESYSTEM;
case LibraryBackend.FlatFilesystem:
return VersionResponse_LibrarySource_LibraryBackend.FLAT_FILESYSTEM;
}
};
return {
type: TorrentialBoundType.VERSION_RESPONSE,
schema: VersionResponseSchema,
data: create(VersionResponseSchema, {
manifest: create(VersionResponse_ManifestSchema, {
version: manifest.version,
size: BigInt(manifest.size),
key: Buffer.from(manifest.key),
chunks: Object.fromEntries(
Object.entries(manifest.chunks).map(([id, chunk]) => [
id,
create(VersionResponse_Manifest_ChunkDataSchema, {
checksum: chunk.checksum,
iv: Buffer.from(chunk.iv),
files: chunk.files.map((file) =>
create(VersionResponse_Manifest_ChunkData_FileEntrySchema, {
filename: file.filename,
start: BigInt(file.start),
length: BigInt(file.length),
permissions: file.permissions,
}),
),
}),
]),
),
}),
source: create(VersionResponse_LibrarySourceSchema, {
options: JSON.stringify(version.game.library.options),
id: version.game.library.id,
backend: mapEnum(version.game.library.backend),
}),
libraryPath: version.game.libraryPath,
versionPath: version.versionPath!,
}),
};
},
});
server/internal/services/torrential/server-games.ts
+38
View File
@@ -0,0 +1,38 @@
import prisma from "../../db/database";
import { ServerGamesResponseSchema } from "../../proto/torrential/proto/manifest_pb";
import { create } from "@bufbuild/protobuf";
import { defineQueryProcessor } from "./utils";
import {
DropBoundType,
TorrentialBoundType,
} from "../../proto/torrential/proto/core_pb";
export default defineQueryProcessor({
queryType: DropBoundType.SERVER_GAMES_QUERY,
run: async () => {
// const queryData = fromBinary(ServerGamesQuerySchema, query.data);
const games = await prisma.game.findMany({
select: {
id: true,
versions: {
select: {
versionId: true,
},
where: {
versionPath: {
not: null,
},
},
},
},
});
return {
type: TorrentialBoundType.SERVER_GAMES_RESPONSE,
schema: ServerGamesResponseSchema,
data: create(ServerGamesResponseSchema, {
games,
}),
};
},
});
server/internal/services/torrential/utils.ts
+15
View File
@@ -0,0 +1,15 @@
import type { Message } from "@bufbuild/protobuf";
import type { QueryProcessor } from ".";
import type {
DropBoundType,
TorrentialBoundType,
} from "../../proto/torrential/proto/core_pb";
export function defineQueryProcessor<
T extends DropBoundType,
K extends TorrentialBoundType,
V extends Message,
>(opts: QueryProcessor<T, K, V>) {
// TORRENTIAL_SERVICE.queryProcessors.set(opts.queryType, opts as any);
return opts;
}
server/internal/tasks/index.ts
+4 -3
View File
@@ -1,4 +1,3 @@
import droplet from "@drop-oss/droplet";
import type { MinimumRequestObject } from "~/server/h3";
import type { GlobalACL } from "../acls";
import aclManager from "../acls";
@@ -212,7 +211,7 @@ class TaskHandler {
await updateAllClients(true);
droplet.callAltThreadFunc(async () => {
const taskFunc = async () => {
const taskEntry = this.taskPool.get(task.id);
if (!taskEntry) throw new Error("No task entry");
const addAction = (action: TaskActionLink) => {
@@ -260,7 +259,9 @@ class TaskHandler {
});
this.taskPool.delete(task.id);
});
};
taskFunc();
return task.id;
}
server/plugins/04.auth-init.ts
+34
View File
@@ -1,5 +1,39 @@
import authManager from "~/server/internal/auth";
import prisma from "../internal/db/database";
import { APITokenMode } from "~/prisma/client/enums";
import type { UserACL } from "../internal/acls";
export const CLIENT_WEBTOKEN_ACLS: UserACL = [
"read",
"store:read",
"object:read",
"settings:read",
"collections:read",
"collections:new",
"collections:add",
"collections:remove",
"collections:delete",
"library:add",
"library:remove",
];
export default defineNitroPlugin(async () => {
await authManager.init();
await prisma.aPIToken.updateMany({
where: {
mode: APITokenMode.Client,
},
data: {
acls: CLIENT_WEBTOKEN_ACLS,
},
});
await prisma.aPIToken.deleteMany({
where: {
id: "torrential",
},
});
});
server/plugins/06.service-spinup.ts
+1 -1
View File
@@ -1,6 +1,6 @@
import serviceManager from "../internal/services";
import { NGINX_SERVICE } from "../internal/services/services/nginx";
import { TORRENTIAL_SERVICE } from "../internal/services/services/torrential";
import { TORRENTIAL_SERVICE } from "../internal/services/torrential";
export default defineNitroPlugin(async (nitro) => {
TORRENTIAL_SERVICE.register();