From 8ee485feef94fc3fa38c2a3d163fb976fa3f8442 Mon Sep 17 00:00:00 2001 From: Huskydog9988 <39809509+Huskydog9988@users.noreply.github.com> Date: Thu, 8 May 2025 11:38:09 -0400 Subject: [PATCH 01/10] feat: basic screenshot manager --- .../migration.sql | 20 +++++++ server/prisma/models/content.prisma | 17 ++++++ server/prisma/models/user.prisma | 3 +- server/server/internal/saves/index.ts | 4 +- server/server/internal/screenshots/index.ts | 58 +++++++++++++++++++ 5 files changed, 99 insertions(+), 3 deletions(-) create mode 100644 server/prisma/migrations/20250508153613_add_screenshots/migration.sql create mode 100644 server/server/internal/screenshots/index.ts diff --git a/server/prisma/migrations/20250508153613_add_screenshots/migration.sql b/server/prisma/migrations/20250508153613_add_screenshots/migration.sql new file mode 100644 index 00000000..08869afb --- /dev/null +++ b/server/prisma/migrations/20250508153613_add_screenshots/migration.sql @@ -0,0 +1,20 @@ +-- CreateTable +CREATE TABLE "Screenshot" ( + "id" TEXT NOT NULL, + "gameId" TEXT NOT NULL, + "userId" TEXT NOT NULL, + "objectId" TEXT NOT NULL, + "private" BOOLEAN NOT NULL DEFAULT true, + "createdAt" TIMESTAMPTZ(0) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "Screenshot_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE INDEX "Screenshot_gameId_userId_idx" ON "Screenshot"("gameId", "userId"); + +-- AddForeignKey +ALTER TABLE "Screenshot" ADD CONSTRAINT "Screenshot_gameId_fkey" FOREIGN KEY ("gameId") REFERENCES "Game"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "Screenshot" ADD CONSTRAINT "Screenshot_userId_fkey" FOREIGN KEY ("userId") REFERENCES "User"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/server/prisma/models/content.prisma b/server/prisma/models/content.prisma index 3264c78a..c321c560 100644 --- a/server/prisma/models/content.prisma +++ b/server/prisma/models/content.prisma @@ -35,6 +35,7 @@ model Game { collections CollectionEntry[] saves SaveSlot[] + screenshots Screenshot[] @@unique([metadataSource, metadataId], name: "metadataKey") } @@ -85,6 +86,22 @@ model SaveSlot { @@id([gameId, userId, index], name: "id") } +model Screenshot { + id String @id @default(uuid()) + + gameId String + game Game @relation(fields: [gameId], references: [id], onDelete: Cascade) + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + objectId String + private Boolean @default(true) + + createdAt DateTime @default(now()) @db.Timestamptz(0) + + @@index([gameId, userId]) +} + model Developer { id String @id @default(uuid()) diff --git a/server/prisma/models/user.prisma b/server/prisma/models/user.prisma index 823bdd05..9a578a1a 100644 --- a/server/prisma/models/user.prisma +++ b/server/prisma/models/user.prisma @@ -17,7 +17,8 @@ model User { tokens APIToken[] sessions Session[] - saves SaveSlot[] + saves SaveSlot[] + screenshots Screenshot[] } model Notification { diff --git a/server/server/internal/saves/index.ts b/server/server/internal/saves/index.ts index 3d41035c..69e4d11c 100644 --- a/server/server/internal/saves/index.ts +++ b/server/server/internal/saves/index.ts @@ -1,9 +1,9 @@ -import Stream from "stream"; +import Stream from "node:stream"; import prisma from "../db/database"; import { applicationSettings } from "../config/application-configuration"; import objectHandler from "../objects"; import { randomUUID, createHash } from "node:crypto"; -import type { IncomingMessage } from "http"; +import type { IncomingMessage } from "node:http"; class SaveManager { async deleteObjectFromSave( diff --git a/server/server/internal/screenshots/index.ts b/server/server/internal/screenshots/index.ts new file mode 100644 index 00000000..e8b4911b --- /dev/null +++ b/server/server/internal/screenshots/index.ts @@ -0,0 +1,58 @@ +import { randomUUID } from "node:crypto"; +import type { IncomingMessage } from "node:http"; +import objectHandler from "../objects"; +import stream from "node:stream/promises"; +import prisma from "../db/database"; + +class ScreenshotManager { + async get(id: string) { + return await prisma.screenshot.findUnique({ + where: { + id, + }, + }); + } + + async getAllByGame(gameId: string, userId: string) { + const results = await prisma.screenshot.findMany({ + where: { + gameId, + userId, + }, + }); + return results; + } + + async delete(id: string) { + await prisma.screenshot.delete({ + where: { + id, + }, + }); + } + + async upload(gameId: string, userId: string, inputStream: IncomingMessage) { + const objectId = randomUUID(); + const saveStream = await objectHandler.createWithStream(objectId, {}, []); + if (!saveStream) + throw createError({ + statusCode: 500, + statusMessage: "Failed to create writing stream to storage backend.", + }); + + // pipe into object store + await stream.pipeline(inputStream, saveStream); + + // TODO: set createAt to the time screenshot was taken + await prisma.screenshot.create({ + data: { + gameId, + userId, + objectId, + }, + }); + } +} + +export const screenshotManager = new ScreenshotManager(); +export default screenshotManager; From 74a54f14366494c813edcdac4ec93978d11ae405 Mon Sep 17 00:00:00 2001 From: Huskydog9988 <39809509+Huskydog9988@users.noreply.github.com> Date: Thu, 8 May 2025 11:57:13 -0400 Subject: [PATCH 02/10] fix: missing metadata preventing game import when a metadata provider fails to import a game's developer / publisher, the import is no longer blocked. the imports usally fail because there isn't a page for these compaines --- server/server/internal/metadata/giantbomb.ts | 8 +++++-- server/server/internal/metadata/igdb.ts | 16 +++++++++----- server/server/internal/metadata/index.ts | 22 ++++++++++++------- .../server/internal/metadata/pcgamingwiki.ts | 10 ++++++--- server/server/internal/metadata/types.d.ts | 4 ++-- 5 files changed, 40 insertions(+), 20 deletions(-) diff --git a/server/server/internal/metadata/giantbomb.ts b/server/server/internal/metadata/giantbomb.ts index 9ff186ae..2fa4d09e 100644 --- a/server/server/internal/metadata/giantbomb.ts +++ b/server/server/internal/metadata/giantbomb.ts @@ -175,14 +175,18 @@ export class GiantBombProvider implements MetadataProvider { const publishers: Publisher[] = []; if (gameData.publishers) { for (const pub of gameData.publishers) { - publishers.push(await publisher(pub.name)); + const res = await publisher(pub.name); + if (res === undefined) continue; + publishers.push(res); } } const developers: Developer[] = []; if (gameData.developers) { for (const dev of gameData.developers) { - developers.push(await developer(dev.name)); + const res = await developer(dev.name); + if (res === undefined) continue; + developers.push(res); } } diff --git a/server/server/internal/metadata/igdb.ts b/server/server/internal/metadata/igdb.ts index d3682ecc..ed7718aa 100644 --- a/server/server/internal/metadata/igdb.ts +++ b/server/server/internal/metadata/igdb.ts @@ -334,10 +334,16 @@ export class IGDBProvider implements MetadataProvider { for (const company of findCompanyResponse) { // if company was a dev or publisher // CANNOT use else since a company can be both - if (foundInvolved.developer) - developers.push(await developer(company.name)); - if (foundInvolved.publisher) - publishers.push(await publisher(company.name)); + if (foundInvolved.developer) { + const res = await developer(company.name); + if (res === undefined) continue; + developers.push(res); + } + if (foundInvolved.publisher) { + const res = await publisher(company.name); + if (res === undefined) continue; + publishers.push(res); + } } } } @@ -403,7 +409,7 @@ export class IGDBProvider implements MetadataProvider { return metadata; } - throw new Error("No results found"); + throw new Error(`igdb failed to find publisher/developer ${query}`); } async fetchDeveloper( params: _FetchDeveloperMetadataParams, diff --git a/server/server/internal/metadata/index.ts b/server/server/internal/metadata/index.ts index daddd656..b16edbe5 100644 --- a/server/server/internal/metadata/index.ts +++ b/server/server/internal/metadata/index.ts @@ -39,10 +39,10 @@ export abstract class MetadataProvider { abstract fetchGame(params: _FetchGameMetadataParams): Promise; abstract fetchPublisher( params: _FetchPublisherMetadataParams, - ): Promise; + ): Promise; abstract fetchDeveloper( params: _FetchDeveloperMetadataParams, - ): Promise; + ): Promise; } export class MetadataHandler { @@ -192,7 +192,7 @@ export class MetadataHandler { query, "fetchDeveloper", "developer", - )) as Developer; + )) as Developer | undefined; } async fetchPublisher(query: string) { @@ -200,7 +200,7 @@ export class MetadataHandler { query, "fetchPublisher", "publisher", - )) as Publisher; + )) as Publisher | undefined; } // Careful with this function, it has no typechecking @@ -226,9 +226,14 @@ export class MetadataHandler { {}, ["internal:read"], ); - let result: PublisherMetadata; + let result: PublisherMetadata | undefined; try { result = await provider[functionName]({ query, createObject }); + if (result === undefined) { + throw new Error( + `${provider.source()} failed to find a ${databaseName} for "${query}`, + ); + } } catch (e) { console.warn(e); dumpObjects(); @@ -257,9 +262,10 @@ export class MetadataHandler { return object; } - throw new Error( - `No metadata provider found a ${databaseName} for "${query}"`, - ); + // throw new Error( + // `No metadata provider found a ${databaseName} for "${query}"`, + // ); + return undefined; } } diff --git a/server/server/internal/metadata/pcgamingwiki.ts b/server/server/internal/metadata/pcgamingwiki.ts index 7e94f678..e2511ec1 100644 --- a/server/server/internal/metadata/pcgamingwiki.ts +++ b/server/server/internal/metadata/pcgamingwiki.ts @@ -171,7 +171,9 @@ export class PCGamingWikiProvider implements MetadataProvider { if (game.Publishers !== null) { const pubListClean = this.parseCompanyStr(game.Publishers); for (const pub of pubListClean) { - publishers.push(await publisher(pub)); + const res = await publisher(pub); + if (res === undefined) continue; + publishers.push(res); } } @@ -179,7 +181,9 @@ export class PCGamingWikiProvider implements MetadataProvider { if (game.Developers !== null) { const devListClean = this.parseCompanyStr(game.Developers); for (const dev of devListClean) { - developers.push(await developer(dev)); + const res = await developer(dev); + if (res === undefined) continue; + developers.push(res); } } @@ -249,7 +253,7 @@ export class PCGamingWikiProvider implements MetadataProvider { return metadata; } - throw new Error("Error in pcgamingwiki, no publisher"); + throw new Error(`pcgamingwiki failed to find publisher/developer ${query}`); } async fetchDeveloper( diff --git a/server/server/internal/metadata/types.d.ts b/server/server/internal/metadata/types.d.ts index 97db57d5..251060ae 100644 --- a/server/server/internal/metadata/types.d.ts +++ b/server/server/internal/metadata/types.d.ts @@ -57,8 +57,8 @@ export interface _FetchGameMetadataParams { id: string; name: string; - publisher: (query: string) => Promise; - developer: (query: string) => Promise; + publisher: (query: string) => Promise; + developer: (query: string) => Promise; createObject: (data: TransactionDataType) => ObjectReference; } From 74aeb67aeeae1ad4069055cce1a242df5a697db6 Mon Sep 17 00:00:00 2001 From: Huskydog9988 <39809509+Huskydog9988@users.noreply.github.com> Date: Thu, 8 May 2025 12:16:12 -0400 Subject: [PATCH 03/10] feat: manually edit search term for game import --- server/pages/admin/library/import.vue | 35 ++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/server/pages/admin/library/import.vue b/server/pages/admin/library/import.vue index 222d3ee0..157f967c 100644 --- a/server/pages/admin/library/import.vue +++ b/server/pages/admin/library/import.vue @@ -95,6 +95,30 @@
+
+
+ +
+ +
+
+ + Search +
+ (); +const gameSearchTerm = ref(""); +const gameSearchLoading = ref(false); async function updateSelectedGame(value: number) { if (currentlySelectedGame.value == value) return; @@ -255,11 +281,18 @@ async function updateSelectedGame(value: number) { metadataResults.value = undefined; currentlySelectedMetadata.value = -1; + gameSearchTerm.value = game; + await searchGame(); +} + +async function searchGame() { + gameSearchLoading.value = true; const results = await $dropFetch( - `/api/v1/admin/import/game/search?q=${encodeURIComponent(game)}`, + `/api/v1/admin/import/game/search?q=${encodeURIComponent(gameSearchTerm.value)}`, ); metadataResults.value = results; + gameSearchLoading.value = false; } function updateSelectedGame_wrapper(value: number) { From 75f48437f2287a6a234ef27ee60314baaa70368b Mon Sep 17 00:00:00 2001 From: Huskydog9988 <39809509+Huskydog9988@users.noreply.github.com> Date: Thu, 8 May 2025 19:19:10 -0400 Subject: [PATCH 04/10] feat: identify unused objects --- server/server/internal/objects/fsBackend.ts | 3 + .../server/internal/objects/objectHandler.ts | 8 ++ server/server/tasks/cleanup/objects.ts | 121 ++++++++++++++++++ 3 files changed, 132 insertions(+) create mode 100644 server/server/tasks/cleanup/objects.ts diff --git a/server/server/internal/objects/fsBackend.ts b/server/server/internal/objects/fsBackend.ts index a3e33fe6..3f0b8651 100644 --- a/server/server/internal/objects/fsBackend.ts +++ b/server/server/internal/objects/fsBackend.ts @@ -152,6 +152,9 @@ export class FsObjectBackend extends ObjectBackend { await store.save(id, hashResult); return typeof hashResult; } + async listAll(): Promise { + return fs.readdirSync(this.baseObjectPath); + } } class FsHashStore { diff --git a/server/server/internal/objects/objectHandler.ts b/server/server/internal/objects/objectHandler.ts index 6fe9afe7..edbf8ad2 100644 --- a/server/server/internal/objects/objectHandler.ts +++ b/server/server/internal/objects/objectHandler.ts @@ -65,6 +65,7 @@ export abstract class ObjectBackend { metadata: ObjectMetadata, ): Promise; abstract fetchHash(id: ObjectReference): Promise; + abstract listAll(): Promise; } export class ObjectHandler { @@ -244,4 +245,11 @@ export class ObjectHandler { async deleteAsSystem(id: ObjectReference) { return await this.backend.delete(id); } + + /** + * List all objects + */ + async listAll() { + return await this.backend.listAll(); + } } diff --git a/server/server/tasks/cleanup/objects.ts b/server/server/tasks/cleanup/objects.ts new file mode 100644 index 00000000..5c3e7190 --- /dev/null +++ b/server/server/tasks/cleanup/objects.ts @@ -0,0 +1,121 @@ +import prisma from "~/server/internal/db/database"; +import objectHandler from "~/server/internal/objects"; + +type FieldReferenceMap = { + [modelName: string]: { + model: unknown; // Prisma model + fields: string[]; // Fields that may contain IDs + arrayFields: string[]; // Fields that are arrays that may contain IDs + }; +}; + +export default defineTask({ + meta: { + name: "cleanup:objects", + }, + async run() { + console.log("[Task cleanup:objects]: Cleaning unreferenced objects"); + + const objects = await objectHandler.listAll(); + console.log( + `[Task cleanup:objects]: searching for ${objects.length} objects`, + ); + console.log(objects); + const results = await findUnreferencedStrings(objects, buildRefMap()); + console.log("[Task cleanup:objects]: Unreferenced objects: ", results); + + console.log("[Task cleanup:objects]: Done"); + return { result: true }; + }, +}); + +function buildRefMap(): FieldReferenceMap { + const tables = Object.keys(prisma).filter( + (v) => !(v.startsWith("$") || v.startsWith("_") || v === "constructor"), + ); + // type test = Prisma.ModelName + // prisma.game.fields.mIconId. + + const result: FieldReferenceMap = {}; + + for (const model of tables) { + // @ts-expect-error can't get model to typematch key names + const fields = Object.keys(prisma[model]["fields"]); + + const single = fields.filter((v) => v.toLowerCase().endsWith("objectid")); + const array = fields.filter((v) => v.toLowerCase().endsWith("objectids")); + + result[model] = { + // @ts-expect-error im not dealing with this + model: prisma[model], + fields: single, + arrayFields: array, + }; + } + + return result; +} + +async function isReferencedInModelFields( + id: string, + fieldRefMap: FieldReferenceMap, +): Promise { + for (const { model, fields, arrayFields } of Object.values(fieldRefMap)) { + const singleFieldOrConditions = fields + ? fields.map((field) => ({ + [field]: { + equals: id, + }, + })) + : []; + const arrayFieldOrConditions = arrayFields + ? arrayFields.map((field) => ({ + [field]: { + has: id, + }, + })) + : []; + + // prisma.game.findFirst({ + // where: { + // OR: [ + // // single item + // { + // mIconId: { + // equals: "", + // }, + // }, + // // array + // { + // mImageCarousel: { + // has: "", + // }, + // }, + // ], + // }, + // }); + + // @ts-expect-error using unknown because im not typing this mess omg + const found = await model.findFirst({ + where: { OR: [...singleFieldOrConditions, ...arrayFieldOrConditions] }, + }); + + if (found) return true; + } + + return false; +} + +async function findUnreferencedStrings( + objects: string[], + fieldRefMap: FieldReferenceMap, +): Promise { + const unreferenced: string[] = []; + + for (const obj of objects) { + const isRef = await isReferencedInModelFields(obj, fieldRefMap); + if (!isRef) unreferenced.push(obj); + } + + return unreferenced; +} From af5739e3c5a35181e9f3a12a5b5c612af47630a9 Mon Sep 17 00:00:00 2001 From: Huskydog9988 <39809509+Huskydog9988@users.noreply.github.com> Date: Thu, 8 May 2025 19:20:34 -0400 Subject: [PATCH 05/10] feat: minimum support for unrefed object cleanup --- server/components/GamePanel.vue | 4 +- server/components/LibraryDirectory.vue | 2 +- server/pages/admin/library/[id]/index.vue | 61 ++++++++++--------- server/pages/admin/library/index.vue | 2 +- server/pages/library/game/[id]/index.vue | 9 ++- server/pages/store/[id]/index.vue | 14 +++-- server/pages/store/index.vue | 2 +- .../migration.sql | 10 +++ server/prisma/models/content.prisma | 12 ++-- .../api/v1/admin/game/image/index.delete.ts | 30 ++++----- .../api/v1/admin/game/image/index.post.ts | 2 +- .../server/api/v1/admin/game/metadata.post.ts | 2 +- server/server/api/v1/store/recent.get.ts | 4 +- server/server/internal/library/index.ts | 2 +- server/server/internal/metadata/index.ts | 8 +-- server/server/internal/saves/index.ts | 12 ++-- 16 files changed, 100 insertions(+), 76 deletions(-) create mode 100644 server/prisma/migrations/20250508224553_cleanup_old_objects/migration.sql diff --git a/server/components/GamePanel.vue b/server/components/GamePanel.vue index f762fa29..9aaa9cc4 100644 --- a/server/components/GamePanel.vue +++ b/server/components/GamePanel.vue @@ -9,7 +9,7 @@ class="absolute inset-0 transition-all duration-300 group-hover:scale-110" > diff --git a/server/components/LibraryDirectory.vue b/server/components/LibraryDirectory.vue index b5272b95..c486714d 100644 --- a/server/components/LibraryDirectory.vue +++ b/server/components/LibraryDirectory.vue @@ -34,7 +34,7 @@ class="flex flex-row items-center w-full p-1 rounded-md transition-all duration-200 hover:bg-zinc-800 hover:scale-105 hover:shadow-lg active:scale-95" > diff --git a/server/pages/admin/library/[id]/index.vue b/server/pages/admin/library/[id]/index.vue index e00c9ac6..72fe2ace 100644 --- a/server/pages/admin/library/[id]/index.vue +++ b/server/pages/admin/library/[id]/index.vue @@ -10,7 +10,7 @@ class="flex flex-col lg:flex-row lg:justify-between items-start lg:items-center gap-2" >
- +

{{ game.mName }} @@ -56,7 +56,7 @@

No images added to the carousel yet. @@ -64,7 +64,7 @@ @@ -242,7 +242,7 @@
@@ -251,7 +251,7 @@ class="transition-all lg:opacity-0 lg:group-hover:opacity-100 absolute inset-0 flex flex-col items-center justify-center gap-y-2 bg-zinc-950/50" >
current {{ [ - image === game.mBannerId ? "banner" : undefined, - image === game.mCoverId ? "cover" : undefined, + image === game.mBannerObjectId ? "banner" : undefined, + image === game.mCoverObjectId ? "cover" : undefined, ] .filter((e) => e) .join(" & ") @@ -400,7 +403,7 @@