Database-level multi-library support #48 (#58)

* feat: start of library backends

* feat: update backend routes and create initializer

* feat: add legacy library creation

* fix: resolve frontend type errors

* fix: runtime errors

* fix: lint
This commit is contained in:
DecDuck
2025-06-01 16:05:05 +10:00
committed by GitHub
parent 490afd0bb7
commit 3e5c3678d5
21 changed files with 664 additions and 298 deletions
+4 -3
View File
@@ -33,11 +33,12 @@ export default defineEventHandler(async (h3) => {
},
});
if (!game)
if (!game || !game.libraryId)
throw createError({ statusCode: 404, statusMessage: "Game ID not found" });
const unimportedVersions = await libraryManager.fetchUnimportedVersions(
game.id,
const unimportedVersions = await libraryManager.fetchUnimportedGameVersions(
game.libraryId,
game.libraryPath,
);
return { game, unimportedVersions };
+6 -1
View File
@@ -6,5 +6,10 @@ export default defineEventHandler(async (h3) => {
if (!allowed) throw createError({ statusCode: 403 });
const unimportedGames = await libraryManager.fetchAllUnimportedGames();
return { unimportedGames };
const iterableUnimportedGames = Object.entries(unimportedGames)
.map(([libraryId, gameArray]) =>
gameArray.map((e) => ({ game: e, library: libraryId })),
)
.flat();
return { unimportedGames: iterableUnimportedGames };
});
+38 -29
View File
@@ -1,37 +1,46 @@
import { type } from "arktype";
import { throwingArktype } from "~/server/arktype";
import aclManager from "~/server/internal/acls";
import libraryManager from "~/server/internal/library";
import metadataHandler from "~/server/internal/metadata";
import type {
GameMetadataSearchResult,
GameMetadataSource,
} from "~/server/internal/metadata/types";
export default defineEventHandler(async (h3) => {
const allowed = await aclManager.allowSystemACL(h3, ["import:game:new"]);
if (!allowed) throw createError({ statusCode: 403 });
const ImportGameBody = type({
library: "string",
path: "string",
["metadata?"]: {
id: "string",
sourceId: "string",
name: "string",
},
}).configure(throwingArktype);
const body = await readBody(h3);
export default defineEventHandler<{ body: typeof ImportGameBody.infer }>(
async (h3) => {
const allowed = await aclManager.allowSystemACL(h3, ["import:game:new"]);
if (!allowed) throw createError({ statusCode: 403 });
const path = body.path;
const metadata = body.metadata as GameMetadataSearchResult &
GameMetadataSource;
if (!path)
throw createError({
statusCode: 400,
statusMessage: "Path missing from body",
});
const { library, path, metadata } = await readValidatedBody(
h3,
ImportGameBody,
);
const validPath = await libraryManager.checkUnimportedGamePath(path);
if (!validPath)
throw createError({
statusCode: 400,
statusMessage: "Invalid unimported game path",
});
if (!path)
throw createError({
statusCode: 400,
statusMessage: "Path missing from body",
});
if (!metadata || !metadata.id || !metadata.sourceId) {
console.log(metadata);
return await metadataHandler.createGameWithoutMetadata(path);
} else {
return await metadataHandler.createGame(metadata, path);
}
});
const valid = await libraryManager.checkUnimportedGamePath(library, path);
if (!valid)
throw createError({
statusCode: 400,
statusMessage: "Invalid library or game.",
});
if (!metadata) {
return await metadataHandler.createGameWithoutMetadata(library, path);
} else {
return await metadataHandler.createGame(metadata, library, path);
}
},
);
@@ -1,4 +1,5 @@
import aclManager from "~/server/internal/acls";
import prisma from "~/server/internal/db/database";
import libraryManager from "~/server/internal/library";
export default defineEventHandler(async (h3) => {
@@ -13,8 +14,17 @@ export default defineEventHandler(async (h3) => {
statusMessage: "Missing id in request params",
});
const unimportedVersions =
await libraryManager.fetchUnimportedVersions(gameId);
const game = await prisma.game.findUnique({
where: { id: gameId },
select: { libraryId: true, libraryPath: true },
});
if (!game || !game.libraryId)
throw createError({ statusCode: 404, statusMessage: "Game not found" });
const unimportedVersions = await libraryManager.fetchUnimportedGameVersions(
game.libraryId,
game.libraryPath,
);
if (!unimportedVersions)
throw createError({ statusCode: 400, statusMessage: "Invalid game ID" });
@@ -9,31 +9,31 @@ const ImportVersion = type({
version: "string",
platform: "string",
launch: "string?",
launchArgs: "string?",
setup: "string?",
setupArgs: "string?",
onlySetup: "boolean?",
delta: "boolean?",
umuId: "string?",
launch: "string = ''",
launchArgs: "string = ''",
setup: "string = ''",
setupArgs: "string = ''",
onlySetup: "boolean = false",
delta: "boolean = false",
umuId: "string = ''",
});
export default defineEventHandler(async (h3) => {
const allowed = await aclManager.allowSystemACL(h3, ["import:version:new"]);
if (!allowed) throw createError({ statusCode: 403 });
const body = await readValidatedBody(h3, ImportVersion);
const gameId = body.id;
const versionName = body.version;
const platform = body.platform;
const launch = body.launch ?? "";
const launchArgs = body.launchArgs ?? "";
const setup = body.setup ?? "";
const setupArgs = body.setupArgs ?? "";
const onlySetup = body.onlySetup ?? false;
const delta = body.delta ?? false;
const umuId = body.umuId ?? "";
const {
id,
version,
platform,
launch,
launchArgs,
setup,
setupArgs,
onlySetup,
delta,
umuId,
} = await readValidatedBody(h3, ImportVersion);
const platformParsed = parsePlatform(platform);
if (!platformParsed)
@@ -41,7 +41,7 @@ export default defineEventHandler(async (h3) => {
if (delta) {
const validOverlayVersions = await prisma.gameVersion.count({
where: { gameId: gameId, platform: platformParsed, delta: false },
where: { gameId: id, platform: platformParsed, delta: false },
});
if (validOverlayVersions == 0)
throw createError({
@@ -66,7 +66,7 @@ export default defineEventHandler(async (h3) => {
}
// startup & delta require more complex checking logic
const taskId = await libraryManager.importVersion(gameId, versionName, {
const taskId = await libraryManager.importVersion(id, version, {
platform,
onlySetup,
+46 -27
View File
@@ -1,10 +1,14 @@
import cacheHandler from "~/server/internal/cache";
import prisma from "~/server/internal/db/database";
import fs from "fs";
import path from "path";
import libraryManager from "~/server/internal/library";
const chunkSize = 1024 * 1024 * 64;
const gameLookupCache = cacheHandler.createCache<{
libraryId: string | null;
libraryPath: string;
}>("downloadGameLookupCache");
export default defineEventHandler(async (h3) => {
const query = getQuery(h3);
const gameId = query.id?.toString();
@@ -18,36 +22,40 @@ export default defineEventHandler(async (h3) => {
statusMessage: "Invalid chunk arguments",
});
const game = await prisma.game.findUnique({
where: {
id: gameId,
},
select: {
libraryBasePath: true,
},
});
if (!game)
throw createError({ statusCode: 400, statusMessage: "Invalid game ID" });
let game = await gameLookupCache.getItem(gameId);
if (!game) {
game = await prisma.game.findUnique({
where: {
id: gameId,
},
select: {
libraryId: true,
libraryPath: true,
},
});
if (!game || !game.libraryId)
throw createError({ statusCode: 400, statusMessage: "Invalid game ID" });
const versionDir = path.join(
libraryManager.fetchLibraryPath(),
game.libraryBasePath,
versionName,
);
if (!fs.existsSync(versionDir))
await gameLookupCache.setItem(gameId, game);
}
if (!game.libraryId)
throw createError({
statusCode: 400,
statusMessage: "Invalid version name",
statusCode: 500,
statusMessage: "Somehow, we got here.",
});
const gameFile = path.join(versionDir, filename);
if (!fs.existsSync(gameFile))
throw createError({ statusCode: 400, statusMessage: "Invalid game file" });
const gameFileStats = fs.statSync(gameFile);
const peek = await libraryManager.peekFile(
game.libraryId,
game.libraryPath,
versionName,
filename,
);
if (!peek)
throw createError({ status: 400, statusMessage: "Failed to peek file" });
const start = chunkIndex * chunkSize;
const end = Math.min((chunkIndex + 1) * chunkSize, gameFileStats.size);
const end = Math.min((chunkIndex + 1) * chunkSize, peek.size);
const currentChunkSize = end - start;
setHeader(h3, "Content-Length", currentChunkSize);
@@ -57,7 +65,18 @@ export default defineEventHandler(async (h3) => {
statusMessage: "Invalid chunk index",
});
const gameReadStream = fs.createReadStream(gameFile, { start, end: end - 1 }); // end needs to be offset by 1
const gameReadStream = await libraryManager.readFile(
game.libraryId,
game.libraryPath,
versionName,
filename,
{ start, end: end - 1 },
); // end needs to be offset by 1
if (!gameReadStream)
throw createError({
statusCode: 400,
statusMessage: "Failed to create stream",
});
return sendStream(h3, gameReadStream);
});
+107
View File
@@ -0,0 +1,107 @@
import { ArkErrors, type } from "arktype";
import {
GameNotFoundError,
VersionNotFoundError,
type LibraryProvider,
} from "./provider";
import { LibraryBackend } from "~/prisma/client";
import fs from "fs";
import path from "path";
import droplet from "@drop-oss/droplet";
import type { Readable } from "stream";
export const FilesystemProviderConfig = type({
baseDir: "string",
});
export class FilesystemProvider
implements LibraryProvider<typeof FilesystemProviderConfig.infer>
{
private config: typeof FilesystemProviderConfig.infer;
private myId: string;
constructor(rawConfig: unknown, id: string) {
const config = FilesystemProviderConfig(rawConfig);
if (config instanceof ArkErrors) {
throw new Error(
`Failed to create filesystem provider: ${config.summary}`,
);
}
this.myId = id;
this.config = config;
fs.mkdirSync(this.config.baseDir, { recursive: true });
}
id(): string {
return this.myId;
}
type(): LibraryBackend {
return LibraryBackend.Filesystem;
}
async listGames(): Promise<string[]> {
const dirs = fs.readdirSync(this.config.baseDir);
const folderDirs = dirs.filter((e) => {
const fullDir = path.join(this.config.baseDir, e);
return fs.lstatSync(fullDir).isDirectory();
});
return folderDirs;
}
async listVersions(game: string): Promise<string[]> {
const gameDir = path.join(this.config.baseDir, game);
if (!fs.existsSync(gameDir)) throw new GameNotFoundError();
const versionDirs = fs.readdirSync(gameDir);
const validVersionDirs = versionDirs.filter((e) => {
const fullDir = path.join(this.config.baseDir, game, e);
return droplet.hasBackendForPath(fullDir);
});
return validVersionDirs;
}
async versionReaddir(game: string, version: string): Promise<string[]> {
const versionDir = path.join(this.config.baseDir, game, version);
if (!fs.existsSync(versionDir)) throw new VersionNotFoundError();
return droplet.listFiles(versionDir);
}
async generateDropletManifest(
game: string,
version: string,
progress: (err: Error | null, v: number) => void,
log: (err: Error | null, v: string) => void,
): Promise<string> {
const versionDir = path.join(this.config.baseDir, game, version);
if (!fs.existsSync(versionDir)) throw new VersionNotFoundError();
const manifest = await new Promise<string>((r, j) =>
droplet.generateManifest(versionDir, progress, log, (err, result) => {
if (err) return j(err);
r(result);
}),
);
return manifest;
}
// TODO: move this over to the droplet.readfile function it works
async readFile(
game: string,
version: string,
filename: string,
options?: { start?: number; end?: number },
): Promise<Readable | undefined> {
const filepath = path.join(this.config.baseDir, game, version, filename);
if (!fs.existsSync(filepath)) return undefined;
const stream = fs.createReadStream(filepath, options);
return stream;
}
async peekFile(game: string, version: string, filename: string) {
const filepath = path.join(this.config.baseDir, game, version, filename);
if (!fs.existsSync(filepath)) return undefined;
const stat = fs.statSync(filepath);
return { size: stat.size };
}
}
+104 -114
View File
@@ -5,60 +5,63 @@
* It also provides the endpoints with information about unmatched games
*/
import fs from "fs";
import path from "path";
import prisma from "../db/database";
import type { GameVersion } from "~/prisma/client";
import { fuzzy } from "fast-fuzzy";
import { recursivelyReaddir } from "../utils/recursivedirs";
import taskHandler from "../tasks";
import { parsePlatform } from "../utils/parseplatform";
import droplet from "@drop-oss/droplet";
import notificationSystem from "../notifications";
import { systemConfig } from "../config/sys-conf";
import type { LibraryProvider } from "./provider";
class LibraryManager {
private basePath: string;
private libraries: Map<string, LibraryProvider<unknown>> = new Map();
constructor() {
this.basePath = systemConfig.getLibraryFolder();
fs.mkdirSync(this.basePath, { recursive: true });
}
fetchLibraryPath() {
return this.basePath;
addLibrary(library: LibraryProvider<unknown>) {
this.libraries.set(library.id(), library);
}
async fetchAllUnimportedGames() {
const dirs = fs.readdirSync(this.basePath).filter((e) => {
const fullDir = path.join(this.basePath, e);
return fs.lstatSync(fullDir).isDirectory();
});
const unimportedGames: { [key: string]: string[] } = {};
const validGames = await prisma.game.findMany({
where: {
libraryBasePath: { in: dirs },
},
select: {
libraryBasePath: true,
},
});
const validGameDirs = validGames.map((e) => e.libraryBasePath);
for (const [id, library] of this.libraries.entries()) {
const games = await library.listGames();
const validGames = await prisma.game.findMany({
where: {
libraryId: id,
libraryPath: { in: games },
},
select: {
libraryPath: true,
},
});
const providerUnimportedGames = games.filter(
(e) => validGames.findIndex((v) => v.libraryPath == e) == -1,
);
unimportedGames[id] = providerUnimportedGames;
}
const unregisteredGames = dirs.filter((e) => !validGameDirs.includes(e));
return unregisteredGames;
return unimportedGames;
}
async fetchUnimportedGameVersions(
libraryBasePath: string,
versions: Array<GameVersion>,
) {
const gameDir = path.join(this.basePath, libraryBasePath);
const versionsDirs = fs.readdirSync(gameDir);
const importedVersionDirs = versions.map((e) => e.versionName);
const unimportedVersions = versionsDirs.filter(
(e) => !importedVersionDirs.includes(e),
async fetchUnimportedGameVersions(libraryId: string, libraryPath: string) {
const provider = this.libraries.get(libraryId);
if (!provider) return undefined;
const game = await prisma.game.findUnique({
where: {
libraryKey: {
libraryId,
libraryPath,
},
},
select: {
versions: true,
},
});
if (!game) return undefined;
const versions = await provider.listVersions(libraryPath);
const unimportedVersions = versions.filter(
(e) => game.versions.findIndex((v) => v.versionName == e) == -1,
);
return unimportedVersions;
@@ -73,7 +76,8 @@ class LibraryManager {
mShortDescription: true,
metadataSource: true,
mIconObjectId: true,
libraryBasePath: true,
libraryId: true,
libraryPath: true,
},
orderBy: {
mName: "asc",
@@ -85,60 +89,24 @@ class LibraryManager {
game: e,
status: {
noVersions: e.versions.length == 0,
unimportedVersions: await this.fetchUnimportedGameVersions(
e.libraryBasePath,
e.versions,
),
unimportedVersions: (await this.fetchUnimportedGameVersions(
e.libraryId ?? "",
e.libraryPath,
))!,
},
})),
);
}
async fetchUnimportedVersions(gameId: string) {
const game = await prisma.game.findUnique({
where: { id: gameId },
select: {
versions: {
select: {
versionName: true,
},
},
libraryBasePath: true,
},
});
if (!game) return undefined;
const targetDir = path.join(this.basePath, game.libraryBasePath);
if (!fs.existsSync(targetDir))
throw new Error(
"Game in database, but no physical directory? Something is very very wrong...",
);
const versions = fs.readdirSync(targetDir);
const validVersions = versions.filter((versionDir) => {
const versionPath = path.join(targetDir, versionDir);
const stat = fs.statSync(versionPath);
return stat.isDirectory();
});
const currentVersions = game.versions.map((e) => e.versionName);
const unimportedVersions = validVersions.filter(
(e) => !currentVersions.includes(e),
);
return unimportedVersions;
}
async fetchUnimportedVersionInformation(gameId: string, versionName: string) {
const game = await prisma.game.findUnique({
where: { id: gameId },
select: { libraryBasePath: true, mName: true },
select: { libraryPath: true, libraryId: true, mName: true },
});
if (!game) return undefined;
const targetDir = path.join(
this.basePath,
game.libraryBasePath,
versionName,
);
if (!fs.existsSync(targetDir)) return undefined;
if (!game || !game.libraryId) return undefined;
const library = this.libraries.get(game.libraryId);
if (!library) return undefined;
const fileExts: { [key: string]: string[] } = {
Linux: [
@@ -165,7 +133,7 @@ class LibraryManager {
match: number;
}> = [];
const files = recursivelyReaddir(targetDir, 2);
const files = await library.versionReaddir(game.libraryPath, versionName);
for (const file of files) {
const filename = path.basename(file);
const dotLocation = file.lastIndexOf(".");
@@ -174,10 +142,9 @@ class LibraryManager {
for (const checkExt of checkExts) {
if (checkExt != ext) continue;
const fuzzyValue = fuzzy(filename, game.mName);
const relative = path.relative(targetDir, file);
options.push({
filename: relative,
platform: platform,
filename,
platform,
match: fuzzyValue,
});
}
@@ -190,17 +157,22 @@ class LibraryManager {
}
// Checks are done in least to most expensive order
async checkUnimportedGamePath(targetPath: string) {
const targetDir = path.join(this.basePath, targetPath);
if (!fs.existsSync(targetDir)) return false;
async checkUnimportedGamePath(libraryId: string, libraryPath: string) {
const hasGame =
(await prisma.game.count({ where: { libraryBasePath: targetPath } })) > 0;
(await prisma.game.count({ where: { libraryId, libraryPath } })) > 0;
if (hasGame) return false;
return true;
}
/*
Game creation happens in metadata, because it's primarily a metadata object
async createGame(libraryId: string, libraryPath: string, game: Omit<Game, "libraryId" | "libraryPath">) {
}
*/
async importVersion(
gameId: string,
versionName: string,
@@ -224,12 +196,12 @@ class LibraryManager {
const game = await prisma.game.findUnique({
where: { id: gameId },
select: { mName: true, libraryBasePath: true },
select: { mName: true, libraryId: true, libraryPath: true },
});
if (!game) return undefined;
if (!game || !game.libraryId) return undefined;
const baseDir = path.join(this.basePath, game.libraryBasePath, versionName);
if (!fs.existsSync(baseDir)) return undefined;
const library = this.libraries.get(game.libraryId);
if (!library) return undefined;
taskHandler.create({
id: taskId,
@@ -238,23 +210,18 @@ class LibraryManager {
async run({ progress, log }) {
// First, create the manifest via droplet.
// This takes up 90% of our progress, so we wrap it in a *0.9
const manifest = await new Promise<string>((resolve, reject) => {
droplet.generateManifest(
baseDir,
(err, value) => {
if (err) return reject(err);
progress(value * 0.9);
},
(err, line) => {
if (err) return reject(err);
log(line);
},
(err, manifest) => {
if (err) return reject(err);
resolve(manifest);
},
);
});
const manifest = await library.generateDropletManifest(
game.libraryPath,
versionName,
(err, value) => {
if (err) throw err;
progress(value * 0.9);
},
(err, value) => {
if (err) throw err;
log(value);
},
);
log("Created manifest successfully!");
@@ -315,6 +282,29 @@ class LibraryManager {
return taskId;
}
async peekFile(
libraryId: string,
game: string,
version: string,
filename: string,
) {
const library = this.libraries.get(libraryId);
if (!library) return undefined;
return library.peekFile(game, version, filename);
}
async readFile(
libraryId: string,
game: string,
version: string,
filename: string,
options?: { start?: number; end?: number },
) {
const library = this.libraries.get(libraryId);
if (!library) return undefined;
return library.readFile(game, version, filename, options);
}
}
export const libraryManager = new LibraryManager();
+64
View File
@@ -0,0 +1,64 @@
import type { Readable } from "stream";
import type { LibraryBackend } from "~/prisma/client";
export abstract class LibraryProvider<CFG> {
constructor(_config: CFG, _id: string) {
throw new Error("Library doesn't have a proper constructor");
}
/**
* @returns ID of the current library provider (fs, smb, s3, etc)
*/
abstract type(): LibraryBackend;
/**
* @returns the specific ID of this current provider
*/
abstract id(): string;
/**
* @returns list of (usually) top-level game folder names
*/
abstract listGames(): Promise<string[]>;
/**
* @param game folder name of the game to list versions for
* @returns list of version folder names
*/
abstract listVersions(game: string): Promise<string[]>;
/**
* @param game folder name of the game
* @param version folder name of the version
* @returns recursive list of all files in version, relative to the version folder (e.g. ./setup.exe)
*/
abstract versionReaddir(game: string, version: string): Promise<string[]>;
/**
* @param game folder name of the game
* @param version folder name of the version
* @returns string of JSON of the droplet manifest
*/
abstract generateDropletManifest(
game: string,
version: string,
progress: (err: Error | null, v: number) => void,
log: (err: Error | null, v: string) => void,
): Promise<string>;
abstract peekFile(
game: string,
version: string,
filename: string,
): Promise<{ size: number } | undefined>;
abstract readFile(
game: string,
version: string,
filename: string,
options?: { start?: number; end?: number },
): Promise<Readable | undefined>;
}
export class GameNotFoundError extends Error {}
export class VersionNotFoundError extends Error {}
+9 -10
View File
@@ -97,18 +97,15 @@ export class MetadataHandler {
return successfulResults;
}
async createGameWithoutMetadata(libraryBasePath: string) {
async createGameWithoutMetadata(libraryId: string, libraryPath: string) {
return await this.createGame(
{
id: "",
name: libraryBasePath,
icon: "",
description: "",
year: 0,
name: libraryPath,
sourceId: "manual",
sourceName: "Manual",
},
libraryBasePath,
libraryId,
libraryPath,
);
}
@@ -165,8 +162,9 @@ export class MetadataHandler {
}
async createGame(
result: InternalGameMetadataResult,
libraryBasePath: string,
result: { sourceId: string; id: string; name: string },
libraryId: string,
libraryPath: string,
) {
const provider = this.providers.get(result.sourceId);
if (!provider)
@@ -231,7 +229,8 @@ export class MetadataHandler {
connectOrCreate: this.parseTags(metadata.tags),
},
libraryBasePath,
libraryId,
libraryPath,
},
});
+74
View File
@@ -0,0 +1,74 @@
import { LibraryBackend } from "~/prisma/client";
import prisma from "../internal/db/database";
import type { JsonValue } from "@prisma/client/runtime/library";
import type { LibraryProvider } from "../internal/library/provider";
import type { FilesystemProviderConfig } from "../internal/library/filesystem";
import { FilesystemProvider } from "../internal/library/filesystem";
import libraryManager from "../internal/library";
import path from "path";
const libraryConstructors: {
[key in LibraryBackend]: (
value: JsonValue,
id: string,
) => LibraryProvider<unknown>;
} = {
Filesystem: function (
value: JsonValue,
id: string,
): LibraryProvider<unknown> {
return new FilesystemProvider(value, id);
},
};
export default defineNitroPlugin(async () => {
let successes = 0;
const libraries = await prisma.library.findMany({});
// Add migration handler
const legacyPath = process.env.LIBRARY;
if (legacyPath && libraries.length == 0) {
const options: typeof FilesystemProviderConfig.infer = {
baseDir: path.resolve(legacyPath),
};
const library = await prisma.library.create({
data: {
name: "Auto-created",
backend: LibraryBackend.Filesystem,
options,
},
});
libraries.push(library);
// Update all existing games
await prisma.game.updateMany({
where: {
libraryId: null,
},
data: {
libraryId: library.id,
},
});
}
for (const library of libraries) {
const constructor = libraryConstructors[library.backend];
try {
const provider = constructor(library.options, library.id);
libraryManager.addLibrary(provider);
successes++;
} catch (e) {
console.warn(
`Failed to create library (${library.id}) of type ${library.backend}:\n ${e}`,
);
}
}
if (successes == 0) {
console.warn(
"No library was successfully initialised. Please check for errors. If you have just set up an instance, this is normal.",
);
}
});