jessikitty/drop
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

Paginated admin library & upgrade manifests (#351)

Browse Source 
* feat: new page layout + endpoint

* feat: non-parallel mass import

* feat: paginated admin library

* feat: lint and performance improvement

* feat: library filter util

* feat: link frontend features to backend

* fix: lint

* fix: small fixes

* feat: bump torrential

* fix: lint
This commit is contained in:
DecDuck
2026-02-25 02:17:33 +11:00
committed by GitHub
parent 1ad881721e
commit dbe34684d8
35 changed files with 1823 additions and 416 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/api/v1/admin/game/index.get.ts
-16
View File
@@ -1,16 +0,0 @@
import aclManager from "~/server/internal/acls";
import prisma from "~/server/internal/db/database";
export default defineEventHandler(async (h3) => {
const allowed = await aclManager.allowSystemACL(h3, ["game:read"]);
if (!allowed) throw createError({ statusCode: 403 });
return await prisma.game.findMany({
select: {
id: true,
mName: true,
mShortDescription: true,
mIconObjectId: true,
},
});
});
server/api/v1/admin/import/massversion/index.get.ts
+54
View File
@@ -0,0 +1,54 @@
import aclManager from "~/server/internal/acls";
import prisma from "~/server/internal/db/database";
import { libraryManager } from "~/server/internal/library";
export default defineEventHandler(async (h3) => {
const allowed = await aclManager.allowSystemACL(h3, ["import:version:read"]);
if (!allowed) throw createError({ statusCode: 403 });
const games = await prisma.game.findMany({
select: {
id: true,
mName: true,
mIconObjectId: true,
versions: {
select: {
versionPath: true,
},
},
unimportedGameVersions: {
select: {
id: true,
versionName: true,
},
},
libraryId: true,
libraryPath: true,
},
});
const unimportedVersions = await Promise.all(
games.map(async (v) => ({
id: v.id,
name: v.mName,
icon: v.mIconObjectId,
versions: await libraryManager.fetchUnimportedGameVersions(
v.libraryId,
v.libraryPath,
{
gameId: v.id,
versions: v.versions
.map((v) => v.versionPath)
.filter((v) => v !== null),
depotVersions: v.unimportedGameVersions,
},
),
})),
);
const onlyUnimported = unimportedVersions.filter(
(v) => v.versions && v.versions.length > 0,
);
return onlyUnimported;
});
server/api/v1/admin/import/massversion/index.post.ts
+115
View File
@@ -0,0 +1,115 @@
import { type } from "arktype";
import { readDropValidatedBody, throwingArktype } from "~/server/arktype";
import { aclManager } from "~/server/internal/acls";
import { libraryManager } from "~/server/internal/library";
import { taskHandler, wrapTaskContext } from "~/server/internal/tasks";
import type { Platform } from "~/prisma/client/client";
const MassImport = type({
versions: type({
id: "string",
version: type({
type: "'depot' | 'local'",
identifier: "string",
name: "string",
}),
displayName: "string?",
setupMode: "boolean = false",
}).array(),
}).configure(throwingArktype);
export default defineEventHandler(async (h3) => {
const allowed = await aclManager.allowSystemACL(h3, ["import:version:new"]);
if (!allowed) throw createError({ statusCode: 403 });
const body = await readDropValidatedBody(h3, MassImport);
const taskId = await taskHandler.create({
key: "mass-import",
taskGroup: "import:version",
acls: ["system:import:version:read"],
name: `Mass-importing for ${body.versions.length} versions`,
async run({ progress, logger, addAction }) {
for (
let versionIndex = 0;
versionIndex < body.versions.length;
versionIndex++
) {
const version = body.versions[versionIndex];
const preload = await libraryManager.fetchUnimportedVersionInformation(
version.id,
version.version,
);
if (!preload) {
logger.warn(
`failed to fetch preload information for: ${version.version.name} (${version.version.type})`,
);
continue;
}
const chosenPreload = preload.at(0);
if (!chosenPreload) {
logger.warn(
`failed to find preload information for: ${version.version.name} (${version.version.type}), there were no auto-discovered executables`,
);
continue;
}
const launches: Array<{
platform: Platform;
launch: string;
name: string;
}> = [];
const setups: Array<{ platform: Platform; launch: string }> = [];
if (version.setupMode) {
setups.push({
platform: chosenPreload.platform,
launch: chosenPreload.filename,
});
} else {
launches.push({
platform: chosenPreload.platform,
launch: chosenPreload.filename,
name: "Play",
});
}
logger.info(`importing ${version.version.name}`);
const min = versionIndex / body.versions.length;
const max = (versionIndex + 1) / body.versions.length;
await libraryManager.importVersion(
version.id,
version.version,
{
id: version.id,
version: version.version,
launches,
setups,
onlySetup: version.setupMode,
delta: false,
requiredContent: [],
},
wrapTaskContext(
{
logger,
progress,
addAction,
},
{
min: min * 100,
max: max * 100,
prefix: `${version.version.name}`,
},
),
);
logger.info(`finished import for ${version.version.name}`);
progress(max * 100);
}
},
});
return { taskId };
});
server/api/v1/admin/import/version/index.post.ts
-40
View File
@@ -2,7 +2,6 @@ import { type } from "arktype";
import { Platform } from "~/prisma/client/enums";
import { readDropValidatedBody, throwingArktype } from "~/server/arktype";
import aclManager from "~/server/internal/acls";
import prisma from "~/server/internal/db/database";
import libraryManager from "~/server/internal/library";
export const ImportVersion = type({
@@ -42,45 +41,6 @@ export default defineEventHandler(async (h3) => {
const body = await readDropValidatedBody(h3, ImportVersion);
if (body.delta) {
for (const platformObject of [...body.launches, ...body.setups].filter(
(v, i, a) => a.findIndex((k) => k.platform === v.platform) == i,
)) {
const validOverlayVersions = await prisma.gameVersion.count({
where: {
gameId: body.id,
delta: false,
OR: [
{ launches: { some: { platform: platformObject.platform } } },
{
setups: { some: { platform: platformObject.platform } },
},
],
},
});
if (validOverlayVersions == 0)
throw createError({
statusCode: 400,
statusMessage: `Update mode requires a pre-existing version for platform: ${platformObject.platform}`,
});
}
}
if (body.onlySetup) {
if (body.setups.length == 0)
throw createError({
statusCode: 400,
statusMessage: 'Setup required in "setup mode".',
});
} else {
if (body.launches.length == 0)
throw createError({
statusCode: 400,
statusMessage: "Launch executable is required.",
});
}
// startup & delta require more complex checking logic
const taskId = await libraryManager.importVersion(
body.id,
body.version,
server/api/v1/admin/library/index.get.ts
+118 -5
View File
@@ -1,15 +1,128 @@
import { ArkErrors, type } from "arktype";
import type { SerializeObject } from "nitropack";
import type { Prisma } from "~/prisma/client/client";
import aclManager from "~/server/internal/acls";
import prisma from "~/server/internal/db/database";
import libraryManager from "~/server/internal/library";
import deepmerge from "deepmerge";
const Query = type({
query: "string?",
skip: "string.numeric.parse?",
limit: "string.numeric.parse?",
sort: "'default' | 'newest' | 'recent' | 'name' = 'default'",
order: "'asc' | 'desc' = 'desc'",
"filters?": type("string").pipe((s) => s.split(",")),
});
type FetchArg = Parameters<typeof libraryManager.fetchGamesWithStatus>[0];
export type AdminLibraryGame = SerializeObject<
Awaited<ReturnType<typeof libraryManager.fetchGamesWithStatus>>[number]
>;
export default defineEventHandler(async (h3) => {
const allowed = await aclManager.allowSystemACL(h3, ["library:read"]);
if (!allowed) throw createError({ statusCode: 403 });
const unimportedGames = await libraryManager.fetchUnimportedGames();
const games = await libraryManager.fetchGamesWithStatus();
const libraries = await libraryManager.fetchLibraries();
const query = Query(getQuery(h3));
if (query instanceof ArkErrors)
throw createError({ statusCode: 400, message: query.summary });
// Fetch other library data here
const skip = query.skip
? ({
skip: query.skip,
} satisfies FetchArg)
: undefined;
return { unimportedGames, games, hasLibraries: libraries.length > 0 };
const limit = Math.min(query.limit ?? 24, 50);
const sort: Prisma.GameOrderByWithRelationInput = {};
switch (query.sort) {
case "default":
case "newest":
sort.mReleased = query.order;
break;
case "recent":
sort.created = query.order;
break;
case "name":
sort.mName = query.order;
break;
}
const rawFilters: Array<Prisma.GameFindManyArgs & Prisma.GameCountArgs> = [];
if (query.filters && query.filters.length > 0) {
const filterSet = new Set(query.filters);
if (filterSet.has("version.none")) {
rawFilters.push({
where: {
versions: {
none: {},
},
},
});
}
if (filterSet.has("metadata.featured")) {
rawFilters.push({
where: {
featured: true,
},
});
}
if (filterSet.has("metadata.noCarousel")) {
rawFilters.push({
where: {
OR: [
{
mImageCarouselObjectIds: {
isEmpty: true,
},
},
],
},
});
}
if (filterSet.has("metadata.emptyDescription")) {
rawFilters.push({
where: {
mDescription: "",
},
});
}
}
if (query.query) {
rawFilters.push({
where: {
mName: {
contains: query.query,
mode: "insensitive",
},
},
});
}
const filters =
rawFilters.length > 0
? rawFilters.reduce((a, b) => deepmerge(a, b))
: undefined;
const results = await libraryManager.fetchGamesWithStatus({
...skip,
take: limit,
orderBy: sort,
...filters,
});
// Safety: the type is defined as a union between the where and count args
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const count = await prisma.game.count({ ...(filters as any) });
return { results, count };
});
server/api/v1/admin/library/libraries.get.ts
+12
View File
@@ -0,0 +1,12 @@
import aclManager from "~/server/internal/acls";
import libraryManager from "~/server/internal/library";
export default defineEventHandler(async (h3) => {
const allowed = await aclManager.allowSystemACL(h3, ["library:read"]);
if (!allowed) throw createError({ statusCode: 403 });
const unimportedGames = await libraryManager.fetchUnimportedGames();
const libraries = await libraryManager.fetchLibraries();
return { unimportedGames, hasLibraries: libraries.length > 0 };
});
server/api/v1/admin/task/index.get.ts
+10 -4
View File
@@ -1,6 +1,5 @@
import aclManager from "~/server/internal/acls";
import prisma from "~/server/internal/db/database";
import type { TaskMessage } from "~/server/internal/tasks";
import taskHandler from "~/server/internal/tasks";
export default defineEventHandler(async (h3) => {
@@ -14,7 +13,7 @@ export default defineEventHandler(async (h3) => {
});
const runningTasks = (await taskHandler.runningTasks()).map((e) => e.id);
const historicalTasks = (await prisma.task.findMany({
const historicalTasks = await prisma.task.findMany({
where: {
OR: [
{
@@ -28,8 +27,15 @@ export default defineEventHandler(async (h3) => {
orderBy: {
ended: "desc",
},
take: 10,
})) as Array<TaskMessage>;
select: {
id: true,
name: true,
actions: true,
error: true,
success: true,
},
take: 32,
});
const dailyTasks = await taskHandler.dailyTasks();
const weeklyTasks = await taskHandler.weeklyTasks();
server/api/v1/client/game/[id]/versions.get.ts
+13 -1
View File
@@ -1,3 +1,4 @@
import { ArkErrors, type } from "arktype";
import type { Platform } from "~/prisma/client/enums";
import { defineClientEventHandler } from "~/server/internal/clients/event-handler";
import prisma from "~/server/internal/db/database";
@@ -21,6 +22,10 @@ type VersionDownloadOption = {
}>;
};
const Query = type({
previous: "string?",
});
export default defineClientEventHandler(async (h3) => {
const id = getRouterParam(h3, "id")!;
if (!id)
@@ -29,6 +34,10 @@ export default defineClientEventHandler(async (h3) => {
statusMessage: "No ID in router params",
});
const query = Query(getQuery(h3));
if (query instanceof ArkErrors)
throw createError({ statusCode: 400, message: query.summary });
const rawVersions = await prisma.gameVersion.findMany({
where: {
gameId: id,
@@ -93,7 +102,10 @@ export default defineClientEventHandler(async (h3) => {
}
}
const size = await gameSizeManager.getVersionSize(v.versionId);
const size = await gameSizeManager.getVersionSize(
v.versionId,
query.previous,
);
return platformOptions
.entries()
server/api/v1/client/game/manifest.get.ts
+16 -9
View File
@@ -1,15 +1,22 @@
import { ArkErrors, type } from "arktype";
import { defineClientEventHandler } from "~/server/internal/clients/event-handler";
import { createDownloadManifestDetails } from "~/server/internal/library/manifest/index";
export default defineClientEventHandler(async (h3) => {
const query = getQuery(h3);
const version = query.version?.toString();
if (!version)
throw createError({
statusCode: 400,
statusMessage: "Missing version ID in query",
});
const Query = type({
version: "string",
previous: "string?",
refresh: "string?",
});
const result = await createDownloadManifestDetails(version);
export default defineClientEventHandler(async (h3) => {
const query = Query(getQuery(h3));
if (query instanceof ArkErrors)
throw createError({ statusCode: 400, message: query.summary });
const result = await createDownloadManifestDetails(
query.version,
query.previous,
query.refresh == "true",
);
return result;
});
server/internal/gamesize/index.ts
+16 -6
View File
@@ -21,23 +21,33 @@ class GameSizeManager {
private gameBreakdownCache =
cacheHandler.createCache<GameSizeBreakdown>("gameBreakdown");
private gameVersionSizeCacheKey(versionId: string, previousId?: string) {
return `${versionId}${previousId ? `-from-${previousId}` : ""}`;
}
/***
* Gets the size of the game to the user:
* - installSize: size on disk after install
* - downloadSize: how many bytes are downloaded (but not necessarily stored)
*/
async getVersionSize(versionId: string): Promise<GameVersionSize | null> {
if (await this.gameVersionsSizesCache.has(versionId))
return await this.gameVersionsSizesCache.get(versionId);
async getVersionSize(
versionId: string,
previousId?: string,
): Promise<GameVersionSize | null> {
const key = this.gameVersionSizeCacheKey(versionId, previousId);
if (await this.gameVersionsSizesCache.has(key))
return await this.gameVersionsSizesCache.get(key);
try {
const { downloadSize, installSize } =
await createDownloadManifestDetails(versionId);
const { downloadSize, installSize } = await createDownloadManifestDetails(
versionId,
previousId,
);
const result = {
downloadSize,
installSize,
versionId,
} satisfies GameVersionSize;
await this.gameVersionsSizesCache.set(versionId, result);
await this.gameVersionsSizesCache.set(key, result);
return result;
} catch {
return null;
server/internal/library/index.ts
+234 -150
View File
@@ -8,6 +8,7 @@
import path from "path";
import prisma from "../db/database";
import { fuzzy } from "fast-fuzzy";
import type { TaskRunContext } from "../tasks";
import taskHandler from "../tasks";
import notificationSystem from "../notifications";
import { GameNotFoundError, type LibraryProvider } from "./provider";
@@ -20,6 +21,7 @@ import type { ImportVersion } from "~/server/api/v1/admin/import/version/index.p
import { GameType, type Platform } from "~/prisma/client/enums";
import { castManifest } from "./manifest/utils";
import { Shescape } from "shescape";
import type { Prisma } from "~/prisma/client/client";
export function createGameImportTaskId(libraryId: string, libraryPath: string) {
return createHash("md5")
@@ -125,42 +127,33 @@ class LibraryManager {
async fetchUnimportedGameVersions(
libraryId: string,
libraryPath: string,
noFetchParams?: {
gameId: string;
versions: string[];
depotVersions: { id: string; versionName: string }[];
},
): Promise<UnimportedVersionInformation[] | undefined> {
const provider = this.libraries.get(libraryId);
if (!provider) return undefined;
const game = await prisma.game.findUnique({
where: {
libraryKey: {
libraryId,
libraryPath,
let params = noFetchParams;
if (!params) {
const game = await prisma.game.findUnique({
where: {
libraryKey: {
libraryId,
libraryPath,
},
},
},
select: {
id: true,
versions: true,
},
});
if (!game) return undefined;
try {
const versions = await provider.listVersions(
libraryPath,
game.versions.map((v) => v.versionPath).filter((v) => v !== null),
);
const unimportedVersions = versions
.filter(
(e) =>
game.versions.findIndex((v) => v.versionPath == e) == -1 &&
!taskHandler.hasTaskKey(createVersionImportTaskKey(game.id, e)),
)
.map(
(v) =>
({
type: "local",
name: v,
identifier: v,
}) satisfies UnimportedVersionInformation,
);
select: {
id: true,
versions: {
select: {
versionPath: true,
},
},
},
});
if (!game) return undefined;
const depotVersions = await prisma.unimportedGameVersion.findMany({
where: {
gameId: game.id,
@@ -170,7 +163,38 @@ class LibraryManager {
id: true,
},
});
const mappedDepotVersions = depotVersions.map(
params = {
gameId: game.id,
versions: game.versions
.map((v) => v.versionPath)
.filter((v) => v !== null),
depotVersions: depotVersions,
};
}
try {
const versions = await provider.listVersions(
libraryPath,
params.versions,
);
const unimportedVersions = versions
.filter(
(e) =>
params.versions.findIndex((v) => v == e) == -1 &&
!taskHandler.hasTaskKey(
createVersionImportTaskKey(params.gameId, e),
),
)
.map(
(v) =>
({
type: "local",
name: v,
identifier: v,
}) satisfies UnimportedVersionInformation,
);
const mappedDepotVersions = params.depotVersions.map(
(v) =>
({
type: "depot",
@@ -188,29 +212,37 @@ class LibraryManager {
}
}
async fetchGamesWithStatus() {
async fetchGamesWithStatus(
where: Partial<Omit<Prisma.GameFindManyArgs, "include">>,
) {
const games = await prisma.game.findMany({
...where,
include: {
library: true,
versions: true,
},
orderBy: {
mName: "asc",
unimportedGameVersions: true,
},
});
return await Promise.all(
games.map(async (e) => {
const versions = await this.fetchUnimportedGameVersions(
const unimportedVersions = await this.fetchUnimportedGameVersions(
e.libraryId ?? "",
e.libraryPath,
{
gameId: e.id,
versions: e.versions
.map((v) => v.versionPath)
.filter((v) => v !== null),
depotVersions: e.unimportedGameVersions,
},
);
return {
game: e,
status: versions
status: unimportedVersions
? {
noVersions: e.versions.length == 0,
unimportedVersions: versions,
unimportedVersions: unimportedVersions,
}
: ("offline" as const),
};
@@ -375,9 +407,51 @@ class LibraryManager {
gameId: string,
version: UnimportedVersionInformation,
metadata: typeof ImportVersion.infer,
parentTask?: TaskRunContext,
) {
const taskKey = createVersionImportTaskKey(gameId, version.identifier);
if (metadata.delta) {
for (const platformObject of [
...metadata.launches,
...metadata.setups,
].filter(
(v, i, a) => a.findIndex((k) => k.platform === v.platform) == i,
)) {
const validOverlayVersions = await prisma.gameVersion.count({
where: {
gameId: metadata.id,
delta: false,
OR: [
{ launches: { some: { platform: platformObject.platform } } },
{
setups: { some: { platform: platformObject.platform } },
},
],
},
});
if (validOverlayVersions == 0)
throw createError({
statusCode: 400,
message: `Update mode requires a pre-existing version for platform: ${platformObject.platform}`,
});
}
}
if (metadata.onlySetup) {
if (metadata.setups.length == 0)
throw createError({
statusCode: 400,
message: 'Setup required in "setup mode".',
});
} else {
if (metadata.launches.length == 0)
throw createError({
statusCode: 400,
message: "Launch executable is required.",
});
}
const game = await prisma.game.findUnique({
where: { id: gameId },
select: { mName: true, libraryId: true, libraryPath: true, type: true },
@@ -400,124 +474,134 @@ class LibraryManager {
})
: undefined;
return await taskHandler.create({
key: taskKey,
taskGroup: "import:game",
name: `Importing version ${version.name} for ${game.mName}`,
acls: ["system:import:version:read"],
async run({ progress, logger }) {
let versionPath: string | null = null;
let manifest;
let fileList;
return await taskHandler.create(
{
key: taskKey,
taskGroup: "import:version",
name: `Importing version ${version.name} for ${game.mName}`,
acls: ["system:import:version:read"],
async run({ progress, logger }) {
let versionPath: string | null = null;
let manifest;
let fileList;
if (version.type === "local") {
versionPath = version.identifier;
// First, create the manifest via droplet.
// This takes up 90% of our progress, so we wrap it in a *0.9
if (version.type === "local") {
versionPath = version.identifier;
// First, create the manifest via droplet.
// This takes up 90% of our progress, so we wrap it in a *0.9
manifest = await library.generateDropletManifest(
game.libraryPath,
versionPath,
(value) => {
progress(value * 0.9);
},
(value) => {
logger.info(value);
},
);
fileList = await library.versionReaddir(
game.libraryPath,
versionPath,
);
logger.info("Created manifest successfully!");
} else if (version.type === "depot" && unimportedVersion) {
manifest = castManifest(unimportedVersion.manifest);
fileList = unimportedVersion.fileList;
progress(90);
} else {
throw "Could not find or create manifest for this version.";
}
const currentIndex = await prisma.gameVersion.count({
where: { gameId: gameId },
});
// Then, create the database object
const newVersion = await prisma.gameVersion.create({
data: {
game: {
connect: {
id: gameId,
manifest = await library.generateDropletManifest(
game.libraryPath,
versionPath,
(value) => {
progress(value * 0.9);
},
},
displayName: metadata.displayName ?? null,
versionPath,
dropletManifest: manifest,
fileList,
versionIndex: currentIndex,
delta: metadata.delta,
onlySetup: metadata.onlySetup,
setups: {
createMany: {
data: metadata.setups.map((v) => ({
command: v.launch,
platform: v.platform,
})),
(value) => {
logger.info(value);
},
);
fileList = await library.versionReaddir(
game.libraryPath,
versionPath,
);
logger.info("Created manifest successfully!");
} else if (version.type === "depot" && unimportedVersion) {
manifest = castManifest(unimportedVersion.manifest);
fileList = unimportedVersion.fileList;
progress(90);
} else {
throw "Could not find or create manifest for this version.";
}
const largestIndex = await prisma.gameVersion.findFirst({
where: { gameId: gameId },
orderBy: {
versionIndex: "desc",
},
launches: {
createMany: !metadata.onlySetup
? {
data: metadata.launches.map((v) => ({
name: v.name,
command: v.launch,
platform: v.platform,
...(v.emulatorId && game.type === "Game"
? {
emulatorId: v.emulatorId,
}
: undefined),
emulatorSuggestions:
game.type === "Emulator" ? (v.suggestions ?? []) : [],
})),
}
: { data: [] },
},
},
});
logger.info("Successfully created version!");
notificationSystem.systemPush({
nonce: `version-create-${gameId}-${version}`,
title: `'${game.mName}' ('${version.name}') finished importing.`,
description: `Drop finished importing version ${version.name} for ${game.mName}.`,
actions: [`View|/admin/library/${gameId}`],
acls: ["system:import:version:read"],
});
// Ensure cache is filled (also pre-caches the manifest)
try {
await gameSizeManager.getVersionSize(newVersion.versionId);
} catch (e) {
logger.warn(`Failed to pre-cache game size and manifest: ${e}`);
}
if (version.type === "depot") {
// SAFETY: we can only reach this if the type is depot and identifier is valid
// eslint-disable-next-line drop/no-prisma-delete
await prisma.unimportedGameVersion.delete({
where: {
id: version.identifier,
select: {
versionIndex: true,
},
});
}
progress(100);
const currentIndex = largestIndex ? largestIndex.versionIndex + 1 : 0;
// Then, create the database object
const newVersion = await prisma.gameVersion.create({
data: {
game: {
connect: {
id: gameId,
},
},
displayName: metadata.displayName ?? null,
versionPath,
dropletManifest: manifest,
fileList,
versionIndex: currentIndex,
delta: metadata.delta,
onlySetup: metadata.onlySetup,
setups: {
createMany: {
data: metadata.setups.map((v) => ({
command: v.launch,
platform: v.platform,
})),
},
},
launches: {
createMany: !metadata.onlySetup
? {
data: metadata.launches.map((v) => ({
name: v.name,
command: v.launch,
platform: v.platform,
...(v.emulatorId && game.type === "Game"
? {
emulatorId: v.emulatorId,
}
: undefined),
emulatorSuggestions:
game.type === "Emulator" ? (v.suggestions ?? []) : [],
})),
}
: { data: [] },
},
},
});
logger.info("Successfully created version!");
notificationSystem.systemPush({
nonce: `version-create-${gameId}-${version}`,
title: `'${game.mName}' ('${version.name}') finished importing.`,
description: `Drop finished importing version ${version.name} for ${game.mName}.`,
actions: [`View|/admin/library/${gameId}`],
acls: ["system:import:version:read"],
});
// Ensure cache is filled (also pre-caches the manifest)
try {
await gameSizeManager.getVersionSize(newVersion.versionId);
} catch (e) {
logger.warn(`Failed to pre-cache game size and manifest: ${e}`);
}
if (version.type === "depot") {
// SAFETY: we can only reach this if the type is depot and identifier is valid
// eslint-disable-next-line drop/no-prisma-delete
await prisma.unimportedGameVersion.delete({
where: {
id: version.identifier,
},
});
}
progress(100);
},
},
});
parentTask,
);
}
async peekFile(
server/internal/library/manifest/index.ts
+16 -4
View File
@@ -30,10 +30,12 @@ const manifestCache =
*/
export async function createDownloadManifestDetails(
versionId: string,
previous?: string,
refresh = false,
): Promise<DownloadManifestDetails> {
if ((await manifestCache.has(versionId)) && !refresh)
return (await manifestCache.get(versionId))!;
const manifestKey = `${versionId}${previous ? `-from-${previous}` : ""}`;
if ((await manifestCache.has(manifestKey)) && !refresh)
return (await manifestCache.get(manifestKey))!;
const mainVersion = await prisma.gameVersion.findUnique({
where: { versionId },
select: {
@@ -94,6 +96,10 @@ export async function createDownloadManifestDetails(
let installSize = 0;
let downloadSize = 0;
const existingChunks = previous
? await createDownloadManifestDetails(previous)
: undefined;
// Now that we have our file list, filter the manifests
const manifests = new Map<string, DropletManifest>();
for (const version of versionOrder) {
@@ -105,9 +111,15 @@ export async function createDownloadManifestDetails(
const fileNames = Object.fromEntries(files);
const manifest = castManifest(version.dropletManifest);
const filteredChunks = Object.fromEntries(
Object.entries(manifest.chunks).filter(([, chunkData]) => {
Object.entries(manifest.chunks).filter(([_, chunkData]) => {
//if(existingChunks && existingChunks.manifests[version.versionId]?.chunks?.[chunkId]) return false;
let flag = false;
chunkData.files.forEach((fileEntry) => {
if (
existingChunks &&
existingChunks.fileList[fileEntry.filename] == version.versionId
)
return;
if (fileNames[fileEntry.filename]) {
flag = true;
installSize += fileEntry.length;
@@ -134,7 +146,7 @@ export async function createDownloadManifestDetails(
installSize,
downloadSize,
};
await manifestCache.set(versionId, result);
await manifestCache.set(manifestKey, result);
return result;
}
server/internal/services/index.ts
+6 -3
View File
@@ -107,9 +107,12 @@ export class Service<T> {
if (this.spun) this.launch();
});
serviceProcess.stdout?.on("data", (data) =>
this.logger.info(data.toString().trim()),
);
serviceProcess.stdout?.on("data", (data) => {
const lines = data.toString().trim().split("\n");
for (const line of lines) {
this.logger.info(line);
}
});
serviceProcess.stderr?.on("data", (data) =>
this.logger.error(data.toString().trim()),
server/internal/tasks/group.ts
+3
View File
@@ -14,6 +14,9 @@ export const taskGroups = {
"import:game": {
concurrency: true,
},
"import:version": {
concurrency: true,
},
debug: {
concurrency: true,
},
server/internal/tasks/index.ts
+66 -58
View File
@@ -8,7 +8,7 @@ import checkUpdate from "./registry/update";
import cleanupObjects from "./registry/objects";
import { taskGroups, type TaskGroup } from "./group";
import prisma from "../db/database";
import { type } from "arktype";
import { ArkErrors, type } from "arktype";
import pino from "pino";
import { logger } from "~/server/internal/logging";
import { Writable } from "node:stream";
@@ -76,7 +76,7 @@ class TaskHandler {
this.taskCreators.set(task.taskGroup, task.build);
}
async create(iTask: Omit<Task, "id">) {
async create(iTask: Omit<Task, "id">, parentTask?: TaskRunContext) {
const task: Task = { ...iTask, id: crypto.randomUUID() };
if (this.hasTaskID(task.id))
throw new Error("Task with ID already exists.");
@@ -105,6 +105,7 @@ class TaskHandler {
const updateAllClients = (reset = false) =>
new Promise((r) => {
//if (parentTask) return; // NO-OP if we're a child task
if (updateCollectTimeout) {
updateCollectResolves.push(r);
return;
@@ -148,7 +149,10 @@ class TaskHandler {
write(chunk, encoding, callback) {
try {
// chunk is a stringified JSON log line
const logObj = JSON.parse(chunk.toString());
const logObj = TaskLog(JSON.parse(chunk.toString()));
if (logObj instanceof ArkErrors) {
throw logObj;
}
const taskEntry = taskPool.get(task.id);
if (taskEntry) {
taskEntry.log.push(JSON.stringify(logObj));
@@ -156,43 +160,44 @@ class TaskHandler {
}
} catch (e) {
// fallback: ignore or log error
logger.error("Failed to parse log chunk", {
error: e,
chunk: chunk,
});
logger.error(`Failed to parse log chunk: ${e}, ${chunk}`);
}
callback();
},
});
// Use pino with the custom stream
const taskLogger = pino(
{
// You can configure timestamp, level, etc. here
timestamp: pino.stdTimeFunctions.isoTime,
base: null, // Remove pid/hostname if not needed
formatters: {
level(label) {
return {
level: label,
};
const taskLogger =
parentTask?.logger ??
pino(
{
// You can configure timestamp, level, etc. here
timestamp: pino.stdTimeFunctions.isoTime,
base: null, // Remove pid/hostname if not needed
formatters: {
level(label) {
return {
level: label,
};
},
},
},
},
logStream,
);
logStream,
);
const progress = (progress: number) => {
if (progress < 0 || progress > 100) {
logger.error("Progress must be between 0 and 100", { progress });
return;
}
const taskEntry = this.taskPool.get(task.id);
if (!taskEntry) return;
taskEntry.progress = progress;
// log(`Progress: ${progress}%`);
updateAllClients();
};
const progress =
parentTask?.progress ??
((progress: number) => {
if (progress < 0 || progress > 100) {
logger.error("Progress must be between 0 and 100", { progress });
return;
}
const taskEntry = this.taskPool.get(task.id);
if (!taskEntry) return;
taskEntry.progress = progress;
// log(`Progress: ${progress}%`);
updateAllClients();
});
this.taskPool.set(task.id, {
name: task.name,
@@ -233,35 +238,37 @@ class TaskHandler {
taskEntry.endTime = new Date().toISOString();
await updateAllClients();
for (const clientId of taskEntry.clients.keys()) {
if (!this.clientRegistry.get(clientId)) continue;
this.disconnect(clientId, task.id);
if (!parentTask) {
for (const clientId of taskEntry.clients.keys()) {
if (!this.clientRegistry.get(clientId)) continue;
this.disconnect(clientId, task.id);
}
await prisma.task.create({
data: {
id: task.id,
taskGroup: taskEntry.taskGroup,
name: taskEntry.name,
started: taskEntry.startTime,
ended: taskEntry.endTime,
success: taskEntry.success,
progress: taskEntry.progress,
log: taskEntry.log,
acls: taskEntry.acls,
actions: taskEntry.actions,
...(taskEntry.error ? { error: taskEntry.error } : undefined),
},
});
}
await prisma.task.create({
data: {
id: task.id,
taskGroup: taskEntry.taskGroup,
name: taskEntry.name,
started: taskEntry.startTime,
ended: taskEntry.endTime,
success: taskEntry.success,
progress: taskEntry.progress,
log: taskEntry.log,
acls: taskEntry.acls,
actions: taskEntry.actions,
...(taskEntry.error ? { error: taskEntry.error } : undefined),
},
});
this.taskPool.delete(task.id);
};
taskFunc();
const fnPromise = taskFunc();
if (parentTask) await fnPromise;
return task.id;
}
@@ -511,9 +518,10 @@ interface DropTask {
}
export const TaskLog = type({
timestamp: "string",
message: "string",
time: "string",
msg: "string",
level: "string",
prefix: "string?",
});
// /**