jessikitty/drop
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

Game specialisation & delta versions (#323)

Browse Source 
* feat: game specialisation, auto-guess extensions

* fix: enforce specialisation specific schema at API level

* fix: lint

* feat: partial work on depot endpoints

* feat: bump torrential

* feat: dummy version creation for depot uploads

* fix: lint

* fix: types

* fix: lint

* feat: depot version import

* fix: lint

* fix: remove any type

* fix: lint

* fix: push update interval

* fix: cpu usage calculation

* feat: delta version support

* feat: style tweaks for selectlaunch.vue

* fix: lint
This commit is contained in:
DecDuck
2026-01-23 05:04:38 +00:00
committed by GitHub
parent d8db5b5b85
commit 00adab21c2
46 changed files with 1164 additions and 347 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/internal/acls/descriptions.ts
+3
View File
@@ -108,8 +108,11 @@ export const systemACLDescriptions: ObjectFromList<typeof systemACLs> = {
"settings:update": "Update system settings.",
"depot:read": "Read depot information, and search for games",
"depot:new": "Create a new download depot",
"depot:delete": "Remove a download depot",
"depot:upload:new": "Upload a new version to a depot",
"depot:upload:delete": "Remove a depot version",
"system-data:listen":
"Connect to a websocket to receive system data updates.",
server/internal/acls/index.ts
+3
View File
@@ -47,8 +47,11 @@ export type UserACL = Array<(typeof userACLs)[number]>;
export const systemACLs = [
"setup",
"depot:read",
"depot:new",
"depot:delete",
"depot:upload:new",
"depot:upload:delete",
"auth:read",
"auth:simple:invitation:read",
server/internal/gamesize/index.ts
+1 -1
View File
@@ -69,7 +69,7 @@ class GameSizeManager {
}
const { dropletManifest } = (await prisma.gameVersion.findUnique({
where: { gameId_versionId: { versionId, gameId } },
where: { versionId },
}))!;
return castManifest(dropletManifest).size;
server/internal/library/index.ts
+233 -46
View File
@@ -18,6 +18,8 @@ import type { WorkingLibrarySource } from "~/server/api/v1/admin/library/sources
import gameSizeManager from "~/server/internal/gamesize";
import { TORRENTIAL_SERVICE } from "../services/services/torrential";
import type { ImportVersion } from "~/server/api/v1/admin/import/version/index.post";
import { GameType, type Platform } from "~/prisma/client/enums";
import { castManifest } from "./manifest";
export function createGameImportTaskId(libraryId: string, libraryPath: string) {
return createHash("md5")
@@ -34,6 +36,30 @@ export function createVersionImportTaskKey(
.digest("hex");
}
export interface ExecutorVersionGuess {
type: "executor";
executorId: string;
icon: string;
gameName: string;
versionName: string;
launchName: string;
platform: Platform;
}
export interface PlatformVersionGuess {
platform: Platform;
type: "platform";
}
export type VersionGuess = {
filename: string;
match: number;
} & (PlatformVersionGuess | ExecutorVersionGuess);
export interface UnimportedVersionInformation {
type: "local" | "depot";
name: string;
identifier: string;
}
class LibraryManager {
private libraries: Map<string, LibraryProvider<unknown>> = new Map();
@@ -95,7 +121,10 @@ class LibraryManager {
return unimportedGames;
}
async fetchUnimportedGameVersions(libraryId: string, libraryPath: string) {
async fetchUnimportedGameVersions(
libraryId: string,
libraryPath: string,
): Promise<UnimportedVersionInformation[] | undefined> {
const provider = this.libraries.get(libraryId);
if (!provider) return undefined;
const game = await prisma.game.findUnique({
@@ -115,14 +144,40 @@ class LibraryManager {
try {
const versions = await provider.listVersions(
libraryPath,
game.versions.map((v) => v.versionPath),
game.versions.map((v) => v.versionPath).filter((v) => v !== null),
);
const unimportedVersions = versions.filter(
(e) =>
game.versions.findIndex((v) => v.versionPath == e) == -1 &&
!taskHandler.hasTaskKey(createVersionImportTaskKey(game.id, e)),
const unimportedVersions = versions
.filter(
(e) =>
game.versions.findIndex((v) => v.versionPath == e) == -1 &&
!taskHandler.hasTaskKey(createVersionImportTaskKey(game.id, e)),
)
.map(
(v) =>
({
type: "local",
name: v,
identifier: v,
}) satisfies UnimportedVersionInformation,
);
const depotVersions = await prisma.unimportedGameVersion.findMany({
where: {
gameId: game.id,
},
select: {
versionName: true,
id: true,
},
});
const mappedDepotVersions = depotVersions.map(
(v) =>
({
type: "depot",
name: v.versionName,
identifier: v.id,
}) satisfies UnimportedVersionInformation,
);
return unimportedVersions;
return [...unimportedVersions, ...mappedDepotVersions];
} catch (e) {
if (e instanceof GameNotFoundError) {
logger.warn(e);
@@ -165,10 +220,13 @@ class LibraryManager {
/**
* Fetches recommendations and extra data about the version. Doesn't actually check if it's been imported.
* @param gameId
* @param versionName
* @param versionIdentifier
* @returns
*/
async fetchUnimportedVersionInformation(gameId: string, versionName: string) {
async fetchUnimportedVersionInformation(
gameId: string,
versionIdentifier: Omit<UnimportedVersionInformation, "name">,
) {
const game = await prisma.game.findUnique({
where: { id: gameId },
select: { libraryPath: true, libraryId: true, mName: true },
@@ -178,7 +236,7 @@ class LibraryManager {
const library = this.libraries.get(game.libraryId);
if (!library) return undefined;
const fileExts: { [key: string]: string[] } = {
const fileExts: { [key in Platform]: string[] } = {
Linux: [
// Ext for Unity games
".x86_64",
@@ -196,13 +254,60 @@ class LibraryManager {
],
};
const options: Array<{
filename: string;
platform: string;
match: number;
}> = [];
const executorSuggestions = await prisma.launchConfiguration.findMany({
where: {
executorSuggestions: {
isEmpty: false,
},
gameVersion: {
game: {
type: GameType.Executor,
},
},
},
select: {
executorSuggestions: true,
gameVersion: {
select: {
game: {
select: {
mIconObjectId: true,
mName: true,
},
},
displayName: true,
versionPath: true,
},
},
name: true,
launchId: true,
platform: true,
},
});
const options: Array<VersionGuess> = [];
let files;
if (versionIdentifier.type === "local") {
files = await library.versionReaddir(
game.libraryPath,
versionIdentifier.identifier,
);
} else if (versionIdentifier.type === "depot") {
const unimported = await prisma.unimportedGameVersion.findUnique({
where: {
id: versionIdentifier.identifier,
},
select: {
fileList: true,
},
});
if (!unimported) return undefined;
files = unimported.fileList;
} else {
return undefined;
}
const files = await library.versionReaddir(game.libraryPath, versionName);
for (const filename of files) {
const basename = path.basename(filename);
const dotLocation = filename.lastIndexOf(".");
@@ -213,12 +318,32 @@ class LibraryManager {
if (checkExt != ext) continue;
const fuzzyValue = fuzzy(basename, game.mName);
options.push({
type: "platform",
filename: filename.replaceAll(" ", "\\ "),
platform,
platform: platform as Platform,
match: fuzzyValue,
});
}
}
for (const executorSuggestion of executorSuggestions) {
for (const suggestion of executorSuggestion.executorSuggestions) {
if (suggestion != ext) continue;
const fuzzyValue = fuzzy(basename, game.mName);
options.push({
type: "executor",
filename: filename.replaceAll(" ", "\\ "),
match: fuzzyValue,
executorId: executorSuggestion.launchId,
icon: executorSuggestion.gameVersion.game.mIconObjectId,
gameName: executorSuggestion.gameVersion.game.mName,
versionName: (executorSuggestion.gameVersion.displayName ??
executorSuggestion.gameVersion.versionPath)!,
launchName: executorSuggestion.name,
platform: executorSuggestion.platform,
});
}
}
}
const sortedOptions = options.sort((a, b) => b.match - a.match);
@@ -247,49 +372,79 @@ class LibraryManager {
async importVersion(
gameId: string,
versionPath: string,
version: UnimportedVersionInformation,
metadata: typeof ImportVersion.infer,
) {
const taskKey = createVersionImportTaskKey(gameId, versionPath);
const taskKey = createVersionImportTaskKey(gameId, version.identifier);
const game = await prisma.game.findUnique({
where: { id: gameId },
select: { mName: true, libraryId: true, libraryPath: true },
select: { mName: true, libraryId: true, libraryPath: true, type: true },
});
if (!game || !game.libraryId) return undefined;
if (game.type === GameType.Redist && !metadata.onlySetup)
throw createError({
statusCode: 400,
message: "Redistributables can only be in setup-only mode.",
});
const library = this.libraries.get(game.libraryId);
if (!library) return undefined;
const unimportedVersion =
version.type === "depot"
? await prisma.unimportedGameVersion.findUnique({
where: { id: version.identifier },
})
: undefined;
return await taskHandler.create({
key: taskKey,
taskGroup: "import:game",
name: `Importing version ${versionPath} for ${game.mName}`,
name: `Importing version ${version.name} for ${game.mName}`,
acls: ["system:import:version:read"],
async run({ progress, logger }) {
// First, create the manifest via droplet.
// This takes up 90% of our progress, so we wrap it in a *0.9
const manifest = await library.generateDropletManifest(
game.libraryPath,
versionPath,
(err, value) => {
if (err) throw err;
progress(value * 0.9);
},
(err, value) => {
if (err) throw err;
logger.info(value);
},
);
let versionPath: string | null = null;
let manifest;
let fileList;
logger.info("Created manifest successfully!");
if (version.type === "local") {
versionPath = version.identifier;
// First, create the manifest via droplet.
// This takes up 90% of our progress, so we wrap it in a *0.9
manifest = await library.generateDropletManifest(
game.libraryPath,
versionPath,
(err, value) => {
if (err) throw err;
progress(value * 0.9);
},
(err, value) => {
if (err) throw err;
logger.info(value);
},
);
fileList = await library.versionReaddir(
game.libraryPath,
versionPath,
);
logger.info("Created manifest successfully!");
} else if (version.type === "depot" && unimportedVersion) {
manifest = castManifest(unimportedVersion.manifest);
fileList = unimportedVersion.fileList;
progress(90);
} else {
throw "Could not find or create manifest for this version.";
}
const currentIndex = await prisma.gameVersion.count({
where: { gameId: gameId },
});
// Then, create the database object
await prisma.gameVersion.create({
const newVersion = await prisma.gameVersion.create({
data: {
game: {
connect: {
@@ -301,6 +456,7 @@ class LibraryManager {
versionPath,
dropletManifest: manifest,
fileList,
versionIndex: currentIndex,
delta: metadata.delta,
@@ -321,9 +477,13 @@ class LibraryManager {
name: v.name,
command: v.launch,
platform: v.platform,
...(v.executorId
? { executorId: v.executorId }
...(v.executorId && game.type === "Game"
? {
executorId: v.executorId,
}
: undefined),
executorSuggestions:
game.type === "Executor" ? (v.suggestions ?? []) : [],
})),
}
: { data: [] },
@@ -333,17 +493,30 @@ class LibraryManager {
logger.info("Successfully created version!");
notificationSystem.systemPush({
nonce: `version-create-${gameId}-${versionPath}`,
title: `'${game.mName}' ('${versionPath}') finished importing.`,
description: `Drop finished importing version ${versionPath} for ${game.mName}.`,
nonce: `version-create-${gameId}-${version}`,
title: `'${game.mName}' ('${version}') finished importing.`,
description: `Drop finished importing version ${version} for ${game.mName}.`,
actions: [`View|/admin/library/${gameId}`],
acls: ["system:import:version:read"],
});
await libraryManager.cacheCombinedGameSize(gameId);
await libraryManager.cacheGameVersionSize(gameId, versionPath);
await libraryManager.cacheGameVersionSize(gameId, newVersion.versionId);
await TORRENTIAL_SERVICE.utils().invalidate(gameId, versionPath);
await TORRENTIAL_SERVICE.utils().invalidate(
gameId,
newVersion.versionId,
);
if (version.type === "depot") {
// SAFETY: we can only reach this if the type is depot and identifier is valid
// eslint-disable-next-line drop/no-prisma-delete
await prisma.unimportedGameVersion.delete({
where: {
id: version.identifier,
},
});
}
progress(100);
},
});
@@ -390,6 +563,20 @@ class LibraryManager {
},
});
await gameSizeManager.deleteGame(gameId);
// Delete all game versions that depended on this game
await prisma.gameVersion.deleteMany({
where: {
launches: {
some: {
executor: {
gameVersion: {
gameId,
},
},
},
},
},
});
}
async getGameVersionSize(
@@ -421,7 +608,7 @@ class LibraryManager {
await gameSizeManager.cacheCombinedGame(game);
}
async cacheGameVersionSize(gameId: string, versionName: string) {
async cacheGameVersionSize(gameId: string, versionId: string) {
const game = await prisma.game.findFirst({
where: { id: gameId },
include: { versions: true },
@@ -429,7 +616,7 @@ class LibraryManager {
if (!game) {
return;
}
await gameSizeManager.cacheGameVersion(game, versionName);
await gameSizeManager.cacheGameVersion(game, versionId);
}
}
server/internal/library/manifest.ts
+4 -4
View File
@@ -1,12 +1,12 @@
import type { JsonValue } from "@prisma/client/runtime/library";
export type Manifest = V2Manifest;
export type DropletManifest = V2Manifest;
export type V2Manifest = {
version: "2";
size: number;
key: number[];
chunks: { [key: string]: V2ChunkData[] };
chunks: { [key: string]: V2ChunkData };
};
export type V2ChunkData = {
@@ -22,6 +22,6 @@ export type V2FileEntry = {
permissions: number;
};
export function castManifest(manifest: JsonValue): Manifest {
return JSON.parse(manifest as string) as Manifest;
export function castManifest(manifest: JsonValue): DropletManifest {
return JSON.parse(manifest as string) as DropletManifest;
}
server/internal/library/manifest/index.ts
+100
View File
@@ -0,0 +1,100 @@
import prisma from "../../db/database";
import { castManifest, type DropletManifest } from "../manifest";
export type DownloadManifestDetails = {
manifests: { [key: string]: DropletManifest };
fileList: { [key: string]: string };
};
function convertMap<T>(map: Map<string, T>): { [key: string]: T } {
return Object.fromEntries(map.entries().toArray());
}
/**
*
* @param gameId Game ID
* @param versionId Version ID
*/
export async function createDownloadManifestDetails(
versionId: string,
): Promise<DownloadManifestDetails> {
const mainVersion = await prisma.gameVersion.findUnique({
where: { versionId },
select: {
versionId: true,
delta: true,
versionIndex: true,
fileList: true,
negativeFileList: true,
gameId: true,
dropletManifest: true,
},
});
if (!mainVersion)
throw createError({ statusCode: 404, message: "Version not found" });
const collectedVersions = [];
let versionIndex = mainVersion.versionIndex;
while (true) {
const nextVersion = await prisma.gameVersion.findFirst({
where: { gameId: mainVersion.gameId, versionIndex: { lt: versionIndex } },
orderBy: {
versionIndex: "desc",
},
select: {
versionId: true,
versionIndex: true,
delta: true,
fileList: true,
negativeFileList: true,
dropletManifest: true,
},
});
if (!nextVersion)
throw createError({
statusCode: 500,
message: "Delta version without version underneath it.",
});
versionIndex = nextVersion.versionIndex;
collectedVersions.push(nextVersion);
if (!nextVersion.delta) break;
}
collectedVersions.reverse();
// Apply fileList in lowest priority to newest priority
const versionOrder = [...collectedVersions, mainVersion];
const fileList = new Map<string, string>();
for (const version of versionOrder) {
for (const file of version.fileList) {
fileList.set(file, version.versionId);
}
for (const negFile of version.negativeFileList) {
fileList.delete(negFile);
}
}
// Now that we have our file list, filter the manifests
const manifests = new Map<string, DropletManifest>();
for (const version of versionOrder) {
const files = fileList
.entries()
.filter(([, versionId]) => version.versionId === versionId)
.toArray();
if (files.length == 0) continue;
const fileNames = Object.fromEntries(files);
const manifest = castManifest(version.dropletManifest);
const filteredChunks = Object.fromEntries(
Object.entries(manifest.chunks).filter(([, chunkData]) =>
chunkData.files.some((fileEntry) => !!fileNames[fileEntry.filename]),
),
);
manifests.set(version.versionId, {
...manifest,
chunks: filteredChunks,
});
}
return { fileList: convertMap(fileList), manifests: convertMap(manifests) };
}
server/internal/metadata/index.ts
+10 -1
View File
@@ -1,4 +1,5 @@
import type { Prisma } from "~/prisma/client/client";
import type { GameType } from "~/prisma/client/enums";
import { MetadataSource } from "~/prisma/client/enums";
import prisma from "../db/database";
import type {
@@ -118,7 +119,11 @@ export class MetadataHandler {
return successfulResults;
}
async createGameWithoutMetadata(libraryId: string, libraryPath: string) {
async createGameWithoutMetadata(
libraryId: string,
libraryPath: string,
type: GameType,
) {
return await this.createGame(
{
id: "",
@@ -127,6 +132,7 @@ export class MetadataHandler {
},
libraryId,
libraryPath,
type,
);
}
@@ -174,6 +180,7 @@ export class MetadataHandler {
result: { sourceId: string; id: string; name: string },
libraryId: string,
libraryPath: string,
type: GameType,
) {
const provider = this.providers.get(result.sourceId);
if (!provider)
@@ -286,6 +293,8 @@ export class MetadataHandler {
libraryId,
libraryPath,
type,
},
});
server/internal/system-data/index.ts
+54 -15
View File
@@ -7,6 +7,33 @@ export type SystemData = {
cpuCores: number;
};
// See https://github.com/oscmejia/os-utils/blob/master/lib/osutils.js
function getCPUInfo() {
const cpus = os.cpus();
let user = 0;
let nice = 0;
let sys = 0;
let idle = 0;
let irq = 0;
for (const cpu in cpus) {
if (!Object.prototype.hasOwnProperty.call(cpus, cpu)) continue;
user += cpus[cpu].times.user;
nice += cpus[cpu].times.nice;
sys += cpus[cpu].times.sys;
irq += cpus[cpu].times.irq;
idle += cpus[cpu].times.idle;
}
const total = user + nice + sys + idle + irq;
return {
idle: idle,
total: total,
};
}
class SystemManager {
// userId to acl to listenerId
private listeners = new Map<
@@ -14,6 +41,20 @@ class SystemManager {
Map<string, { callback: (systemData: SystemData) => void }>
>();
private lastCPUUpdate: { idle: number; total: number } | undefined;
constructor() {
setInterval(() => {
const systemData = this.getSystemData();
if (!systemData) return;
for (const [, map] of this.listeners.entries()) {
for (const [, { callback }] of map.entries()) {
callback(systemData);
}
}
}, 3000);
}
listen(
userId: string,
id: string,
@@ -22,25 +63,17 @@ class SystemManager {
if (!this.listeners.has(userId)) this.listeners.set(userId, new Map());
// eslint-disable-next-line @typescript-eslint/no-extra-non-null-assertion
this.listeners.get(userId)!!.set(id, { callback });
this.pushUpdate(userId, id);
setInterval(() => this.pushUpdate(userId, id), 3000);
}
unlisten(userId: string, id: string) {
this.listeners.get(userId)?.delete(id);
}
private async pushUpdate(userId: string, id: string) {
const listener = this.listeners.get(userId)?.get(id);
if (!listener) {
throw new Error("Failed to catch-up listener: callback does not exist");
}
listener.callback(this.getSystemData());
}
getSystemData(): SystemData {
getSystemData(): SystemData | undefined {
const cpu = this.cpuLoad();
if (!cpu) return undefined;
return {
cpuLoad: this.cpuLoad(),
cpuLoad: cpu * 100,
totalRam: os.totalmem(),
freeRam: os.freemem(),
cpuCores: os.cpus().length,
@@ -48,9 +81,15 @@ class SystemManager {
}
private cpuLoad() {
const [oneMinLoad, _fiveMinLoad, _fiftenMinLoad] = os.loadavg();
const numberCpus = os.cpus().length;
return 100 - ((numberCpus - oneMinLoad) / numberCpus) * 100;
const last = this.lastCPUUpdate;
this.lastCPUUpdate = getCPUInfo();
if (!last) return undefined;
const idle = this.lastCPUUpdate.idle - last.idle;
const total = this.lastCPUUpdate.total - last.total;
const perc = idle / total;
return 1 - perc;
}
}