From 7dc889852d7bef289ec0b2c74fdef1dde06a1aa5 Mon Sep 17 00:00:00 2001 From: DecDuck Date: Wed, 25 Feb 2026 02:17:33 +1100 Subject: [PATCH] Paginated admin library & upgrade manifests (#351) * feat: new page layout + endpoint * feat: non-parallel mass import * feat: paginated admin library * feat: lint and performance improvement * feat: library filter util * feat: link frontend features to backend * fix: lint * fix: small fixes * feat: bump torrential * fix: lint --- server/.vscode/settings.json | 8 +- server/components/GameEditor/Version.vue | 10 + server/components/LogLine.vue | 7 +- server/components/ProgressBar.vue | 2 +- server/i18n/locales/en_us.json | 23 + server/package.json | 3 + server/pages/admin/library/[id]/index.vue | 2 +- server/pages/admin/library/import.vue | 6 + server/pages/admin/library/index.vue | 583 +++++++++++++++--- server/pages/admin/library/mass-import.vue | 363 +++++++++++ server/pages/admin/task/[id]/index.vue | 2 +- server/pages/admin/task/index.vue | 64 +- server/pages/index.vue | 8 + server/pages/store/[id]/index.vue | 2 +- server/pnpm-lock.yaml | 150 +++++ server/pnpm-workspace.yaml | 4 +- .../migration.sql | 15 + server/prisma/models/content.prisma | 2 +- server/server/api/v1/admin/game/index.get.ts | 16 - .../v1/admin/import/massversion/index.get.ts | 54 ++ .../v1/admin/import/massversion/index.post.ts | 115 ++++ .../api/v1/admin/import/version/index.post.ts | 40 -- .../server/api/v1/admin/library/index.get.ts | 123 +++- .../api/v1/admin/library/libraries.get.ts | 12 + server/server/api/v1/admin/task/index.get.ts | 14 +- .../api/v1/client/game/[id]/versions.get.ts | 14 +- .../server/api/v1/client/game/manifest.get.ts | 25 +- server/server/internal/gamesize/index.ts | 22 +- server/server/internal/library/index.ts | 384 +++++++----- .../server/internal/library/manifest/index.ts | 20 +- server/server/internal/services/index.ts | 9 +- server/server/internal/tasks/group.ts | 3 + server/server/internal/tasks/index.ts | 124 ++-- server/torrential | 2 +- server/utils/parseTaskLog.ts | 8 +- 35 files changed, 1823 insertions(+), 416 deletions(-) create mode 100644 server/pages/admin/library/mass-import.vue create mode 100644 server/prisma/migrations/20260224145112_add_non_null_default_to_carousel_object_ids/migration.sql delete mode 100644 server/server/api/v1/admin/game/index.get.ts create mode 100644 server/server/api/v1/admin/import/massversion/index.get.ts create mode 100644 server/server/api/v1/admin/import/massversion/index.post.ts create mode 100644 server/server/api/v1/admin/library/libraries.get.ts diff --git a/server/.vscode/settings.json b/server/.vscode/settings.json index 0570fe39..0660ee64 100644 --- a/server/.vscode/settings.json +++ b/server/.vscode/settings.json @@ -4,11 +4,7 @@ "strings": "on" }, "i18n-ally.extract.autoDetect": true, - "i18n-ally.extract.ignored": [ - "string >= 14", - "string.alphanumeric >= 5", - "/api/v1/admin/import/version/preload?id=${encodeURIComponent(\n gameId,\n )}&version=${encodeURIComponent(version)}" - ], + "i18n-ally.extract.ignored": ["string >= 14", "string.alphanumeric >= 5"], "i18n-ally.extract.ignoredByFiles": { "components/NewsArticleCreateButton.vue": ["[", "`", "Enter"], "pages/admin/library/sources/index.vue": ["Filesystem"], @@ -19,7 +15,7 @@ "i18n-ally.localesPaths": ["i18n", "i18n/locales"], // i18n Ally settings "i18n-ally.sortKeys": true, - "prisma.pinToPrisma6": true, + "prisma.pinToPrisma6": false, "spellchecker.ignoreWordsList": ["mTLS", "Wireguard"], "sqltools.connections": [ { diff --git a/server/components/GameEditor/Version.vue b/server/components/GameEditor/Version.vue index 28165fad..c2696565 100644 --- a/server/components/GameEditor/Version.vue +++ b/server/components/GameEditor/Version.vue @@ -46,6 +46,12 @@ > {{ $t("library.admin.version.table.path") }} + + {{ $t("library.admin.version.table.delta") }} + {{ version.versionPath }} + + {{ version.delta }} + + + @@ -293,8 +536,23 @@ import { ArrowTopRightOnSquareIcon, InformationCircleIcon, StarIcon, + WrenchScrewdriverIcon, + ArrowLongLeftIcon, + ArrowLongRightIcon, + ChevronDownIcon, + FunnelIcon, } from "@heroicons/vue/20/solid"; import { MagnifyingGlassIcon } from "@heroicons/vue/24/outline"; +import type { AdminLibraryGame } from "~/server/api/v1/admin/library/index.get"; +import { + Disclosure, + DisclosureButton, + DisclosurePanel, + Menu, + MenuButton, + MenuItem, + MenuItems, +} from "@headlessui/vue"; const { t } = useI18n(); @@ -306,33 +564,46 @@ useHead({ title: t("library.admin.title"), }); -const searchQuery = ref(""); - -const libraryState = await $dropFetch("/api/v1/admin/library"); -type LibraryStateGame = (typeof libraryState.games)[number]["game"]; - -const toImport = ref( - Object.values(libraryState.unimportedGames).flat().length > 0, +const { unimportedGames, hasLibraries } = await $dropFetch( + "/api/v1/admin/library/libraries", ); -const libraryGames = ref< - Array< - LibraryStateGame & { - status: "online" | "offline"; - hasNotifications?: boolean; - notifications: { - noVersions?: boolean; - toImport?: boolean; - offline?: boolean; - }; - } - > ->( - libraryState.games.map((e) => { +const route = useRoute(); +const router = useRouter(); + +// Hard limit on server +const pageSize = 24; +const currentIndex = ref( + route.query.page ? parseInt(route.query.page.toString()) - 1 : 0, +); +const maxIndex = ref(0); +const maxPages = computed(() => Math.ceil(maxIndex.value / pageSize)); + +const games = ref([]); +const gamesLoading = ref(false); + +const searchQuery = ref(""); + +function nextPage() { + if (currentIndex.value < maxPages.value - 1) { + currentIndex.value++; + } +} + +function previousPage() { + if (currentIndex.value > 0) { + currentIndex.value--; + } +} + +const toImport = ref(Object.values(unimportedGames).flat().length > 0); + +const libraryGames = computed(() => + games.value.map((e) => { if (e.status == "offline") { return { ...e.game, - status: "offline" as const, + status: "offline", hasNotifications: true, notifications: { offline: true, @@ -355,19 +626,6 @@ const libraryGames = ref< }), ); -const filteredLibraryGames = computed(() => - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore excessively deep ts - libraryGames.value.filter((e) => { - if (!searchQuery.value) return true; - const searchQueryLower = searchQuery.value.toLowerCase(); - if (e.mName.toLowerCase().includes(searchQueryLower)) return true; - if (e.mShortDescription.toLowerCase().includes(searchQueryLower)) - return true; - return false; - }), -); - async function deleteGame(id: string) { await $dropFetch(`/api/v1/admin/game/${id}`, { method: "DELETE", @@ -396,4 +654,147 @@ async function featureGame(id: string) { libraryGames.value[gameIndex].featured = !game.featured; gameFeatureLoading.value[game.id] = false; } + +const currentFilters = ref<{ [key: string]: boolean }>({}); + +function createFilterKey( + filter: { value: string }, + subfilter: { value: string }, +) { + return `${filter.value}.${subfilter.value}`; +} + +const filters = computed( + () => + ({ + version: [ + { + value: "none", + label: t("library.admin.nav.filters.version.none"), + }, + /*{ + value: "available", + label: t("library.admin.nav.filters.version.available"), + },*/ + ], + metadata: [ + { + value: "featured", + label: t("library.admin.nav.filters.metadata.featured"), + }, + { + value: "noCarousel", + label: t("library.admin.nav.filters.metadata.noCarousel"), + }, + { + value: "emptyDescription", + label: t("library.admin.nav.filters.metadata.emptyDescription"), + }, + ], + }) as const, +); + +const filterScaffold = computed( + () => + ({ + version: { + title: t("library.admin.nav.filters.version.title"), + value: "version", + // eslint-disable-next-line @typescript-eslint/no-explicit-any + values: filters.value.version as any, + }, + metadata: { + title: t("library.admin.nav.filters.metadata.title"), + value: "metadata", + // eslint-disable-next-line @typescript-eslint/no-explicit-any + values: filters.value.metadata as any, + }, + }) satisfies { + [key in keyof typeof filters.value]: { + title: string; + value: string; + values: Array<{ value: string; label: string }>; + }; + }, +); + +const sorts: Array = [ + { + name: "Default", + param: "default", + }, + { + name: "Newest", + param: "newest", + }, + { + name: "Recently Added", + param: "recent", + }, + { + name: "Name", + param: "name", + }, +]; + +const currentSort = ref(sorts[0].param); +const sortOrder = ref<"asc" | "desc">("desc"); + +function handleSortClick(option: StoreSortOption, event: MouseEvent) { + event.stopPropagation(); + if (currentSort.value === option.param) { + sortOrder.value = sortOrder.value === "asc" ? "desc" : "asc"; + } else { + currentSort.value = option.param; + sortOrder.value = option.param === "name" ? "asc" : "desc"; + } +} + +async function fetchPage() { + gamesLoading.value = true; + const { results, count } = await $dropFetch("/api/v1/admin/library", { + query: { + skip: currentIndex.value * pageSize, + limit: pageSize, + sort: currentSort.value, + order: sortOrder.value, + filters: Object.entries(currentFilters.value) + .filter(([_, enabled]) => enabled) + .map(([name, _]) => name) + .join(","), + query: searchQuery.value ? searchQuery.value : undefined, + }, + failTitle: "Failed to fetch game library", + }); + maxIndex.value = count; + games.value = results; + gamesLoading.value = false; + router.push({ + path: route.path, + query: { + ...route.query, + page: currentIndex.value + 1, + }, + }); +} + +function watchHandler() { + fetchPage(); + document.body.scrollTop = document.documentElement.scrollTop = 0; +} + +watch([currentIndex, currentSort, sortOrder], watchHandler); + +watch(currentFilters, watchHandler, { deep: true }); + +let searchTimeout: NodeJS.Timeout | undefined; +watch(searchQuery, () => { + if (searchTimeout) clearTimeout(searchTimeout); + gamesLoading.value = true; + searchTimeout = setTimeout(() => { + watchHandler(); + }, 80); +}); + +await fetchPage(); diff --git a/server/pages/admin/library/mass-import.vue b/server/pages/admin/library/mass-import.vue new file mode 100644 index 00000000..4e8c4c48 --- /dev/null +++ b/server/pages/admin/library/mass-import.vue @@ -0,0 +1,363 @@ + + + diff --git a/server/pages/admin/task/[id]/index.vue b/server/pages/admin/task/[id]/index.vue index bcdb0a4f..79d52346 100644 --- a/server/pages/admin/task/[id]/index.vue +++ b/server/pages/admin/task/[id]/index.vue @@ -40,7 +40,7 @@ -
    +
    • -
      -
      +
      +

      {{ $t("tasks.admin.completedTasksTitle") }}

      -
        +
        • - +
          +
          +
          +
          + + +
          +
          +

          + {{ task.name }} +

          +
          +
            + {{ name }} +
          + + + + + +
          +
      @@ -120,6 +171,7 @@
      diff --git a/server/pages/store/[id]/index.vue b/server/pages/store/[id]/index.vue index ebd1493a..f7c385d4 100644 --- a/server/pages/store/[id]/index.vue +++ b/server/pages/store/[id]/index.vue @@ -284,7 +284,7 @@ const ratingArray = Array(5) useHead({ title: game.mName, - link: [{ rel: "icon", href: useObject(game.mIconObjectId) }], + // link: [{ rel: "icon", href: useObject(game.mIconObjectId) }], // Favicon doesn't get reset when we navigate off }); diff --git a/server/pnpm-lock.yaml b/server/pnpm-lock.yaml index 206b3caf..2cf8b216 100644 --- a/server/pnpm-lock.yaml +++ b/server/pnpm-lock.yaml @@ -71,6 +71,9 @@ importers: cookie-es: specifier: ^2.0.0 version: 2.0.0 + deepmerge: + specifier: ^4.3.1 + version: 4.3.1 dotenv: specifier: ^17.2.3 version: 17.2.3 @@ -162,6 +165,9 @@ importers: '@bufbuild/protoc-gen-es': specifier: ^2.11.0 version: 2.11.0(@bufbuild/protobuf@2.11.0) + '@golar/vue': + specifier: ^0.0.13 + version: 0.0.13 '@intlify/eslint-plugin-vue-i18n': specifier: ^4.0.1 version: 4.0.1(eslint@9.31.0(jiti@2.6.1))(jsonc-eslint-parser@2.4.0)(vue-eslint-parser@10.2.0(eslint@9.31.0(jiti@2.6.1)))(yaml-eslint-parser@1.3.0) @@ -201,6 +207,9 @@ importers: eslint-config-prettier: specifier: ^10.1.1 version: 10.1.8(eslint@9.31.0(jiti@2.6.1)) + golar: + specifier: ^0.0.13 + version: 0.0.13(@golar/vue@0.0.13) h3: specifier: ^1.15.5 version: 1.15.5 @@ -938,6 +947,48 @@ packages: '@fastify/busboy@3.1.1': resolution: {integrity: sha512-5DGmA8FTdB2XbDeEwc/5ZXBl6UbBAyBOOLlPuBnZ/N1SwdH9Ii+cOX3tBROlDgcTXxjOYnLMVoKk9+FXAw0CJw==} + '@golar/darwin-arm64@0.0.13': + resolution: {integrity: sha512-4W9s7NwtH5goTb2VO/U0JraT5qtdbL7GA9p2mGWK/eMmoTyhoWXEn/ACGC3lI7w7YnzgNXupWL4jMFXFCTmLCA==} + cpu: [arm64] + os: [darwin] + + '@golar/darwin-x64@0.0.13': + resolution: {integrity: sha512-HHoYeFO0nlQiYJWP7LB2qltaPoGjwlltC9YfZvww9SZRmfr2asBUKGywg57YYMN0BZCzT4kUJhQaCld0SUKI+w==} + cpu: [x64] + os: [darwin] + + '@golar/linux-arm64@0.0.13': + resolution: {integrity: sha512-cyLq3PKFTU8wexkhoug2XewiHrLbCp5G6k7X8sFLUNruAAOfQdstc9vz50XUf2FUMUmMbguuuvBth+MhqdEuMg==} + cpu: [arm64] + os: [linux] + + '@golar/linux-x64@0.0.13': + resolution: {integrity: sha512-9+KuD9P3pftqaXpDHZz4+XE/tJUQEZoohluvBe8aaLJpwJ6RlAMIyyumy4H23WA4JbCsDX7kshR/38JNEnjHAg==} + cpu: [x64] + os: [linux] + + '@golar/plugin@0.0.13': + resolution: {integrity: sha512-QL+djNQZh8hpDnFCEt895zL46sUjD42CrBT1Lwrfypvg2pVZFd6acXHos7cfdWt6f2Yb7a+rEnmshjFuZbGcOQ==} + + '@golar/util@0.0.13': + resolution: {integrity: sha512-j4UBvVrOcIRk+1ENQDJjaBJeiSVhWMPDpy+IfTB7Y66vRQgrDZ7LNPsWXooTjnWdaFrNpH9CkJB3ahFkDcfYTw==} + + '@golar/volar@0.0.13': + resolution: {integrity: sha512-MoHPLt7WUuYymb+cizAGW0U3wFE8XcvQI3jpfM7fSn9reAFYEXMXFMLgGAOmnjuunRUFbw46FVqcBx5AoTNyhA==} + + '@golar/vue@0.0.13': + resolution: {integrity: sha512-X84o6B1eJ6ytUbRMOugljG4VyJwXdqD1xt9S6t0YjalhbVEanW4OXq4VPrCzwqGiaTQJTBuVxPUEKq9Bojy9fw==} + + '@golar/win32-arm64@0.0.13': + resolution: {integrity: sha512-aHtHHBOJSfW63oap7KQUpgkIcAOT2iZasoIYb++89iYvo57MeLANpU1sfeFl8y2M9gDlEP5KOwlaLG1FZXwaLQ==} + cpu: [arm64] + os: [win32] + + '@golar/win32-x64@0.0.13': + resolution: {integrity: sha512-JXvb0Qxy/lV1PO35cxpBrlGt/wjlAmQjLHrlMn+y8AriNa34GVYXMfqQoDVlaP8F+hqI9a/PrfDePsPBHILWwg==} + cpu: [x64] + os: [win32] + '@headlessui/vue@1.7.23': resolution: {integrity: sha512-JzdCNqurrtuu0YW6QaDtR2PIYCKPUWq28csDyMvN4zmGccmE7lz40Is6hc3LA4HFeCI7sekZ/PQMTNmn9I/4Wg==} engines: {node: '>=10'} @@ -2492,12 +2543,24 @@ packages: '@volar/language-core@2.4.23': resolution: {integrity: sha512-hEEd5ET/oSmBC6pi1j6NaNYRWoAiDhINbT8rmwtINugR39loROSlufGdYMF9TaKGfz+ViGs1Idi3mAhnuPcoGQ==} + '@volar/language-core@2.4.27': + resolution: {integrity: sha512-DjmjBWZ4tJKxfNC1F6HyYERNHPYS7L7OPFyCrestykNdUZMFYzI9WTyvwPcaNaHlrEUwESHYsfEw3isInncZxQ==} + + '@volar/language-core@2.4.28': + resolution: {integrity: sha512-w4qhIJ8ZSitgLAkVay6AbcnC7gP3glYM3fYwKV3srj8m494E3xtrCv6E+bWviiK/8hs6e6t1ij1s2Endql7vzQ==} + '@volar/source-map@2.4.20': resolution: {integrity: sha512-mVjmFQH8mC+nUaVwmbxoYUy8cww+abaO8dWzqPUjilsavjxH0jCJ3Mp8HFuHsdewZs2c+SP+EO7hCd8Z92whJg==} '@volar/source-map@2.4.23': resolution: {integrity: sha512-Z1Uc8IB57Lm6k7q6KIDu/p+JWtf3xsXJqAX/5r18hYOTpJyBn0KXUR8oTJ4WFYOcDzWC9n3IflGgHowx6U6z9Q==} + '@volar/source-map@2.4.27': + resolution: {integrity: sha512-ynlcBReMgOZj2i6po+qVswtDUeeBRCTgDurjMGShbm8WYZgJ0PA4RmtebBJ0BCYol1qPv3GQF6jK7C9qoVc7lg==} + + '@volar/source-map@2.4.28': + resolution: {integrity: sha512-yX2BDBqJkRXfKw8my8VarTyjv48QwxdJtvRgUpNE5erCsgEUdI2DsLbpa+rOQVAJYshY99szEcRDmyHbF10ggQ==} + '@volar/typescript@2.4.20': resolution: {integrity: sha512-Oc4DczPwQyXcVbd+5RsNEqX6ia0+w3p+klwdZQ6ZKhFjWoBP9PCPQYlKYRi/tDemWphW93P/Vv13vcE9I9D2GQ==} @@ -2592,6 +2655,9 @@ packages: typescript: optional: true + '@vue/language-core@3.2.2': + resolution: {integrity: sha512-5DAuhxsxBN9kbriklh3Q5AMaJhyOCNiQJvCskN9/30XOpdLiqZU9Q+WvjArP17ubdGEyZtBzlIeG5nIjEbNOrQ==} + '@vue/reactivity@3.5.27': resolution: {integrity: sha512-vvorxn2KXfJ0nBEnj4GYshSgsyMNFnIQah/wczXlsNXt+ijhugmW+PpJ2cNPe4V6jpnBcs0MhCODKllWG+nvoQ==} @@ -3857,6 +3923,24 @@ packages: resolution: {integrity: sha512-oB4vkQGqlMl682wL1IlWd02tXCbquGWM4voPEI85QmNKCaw8zGTm1f1rubFgkg3Eli2PtKlFgrnmUqasbQWlkw==} engines: {node: '>=20'} + golar@0.0.13: + resolution: {integrity: sha512-qXAx6MJiuUVCILcwt237R1LbsnqYsgCi0LEcyvuATfKpEaanUTB7jvBlTLKqUK01SmKFd0iCsdG6d+XWPbewiw==} + hasBin: true + peerDependencies: + '@golar/astro': 0.0.13 + '@golar/ember': 0.0.13 + '@golar/svelte': 0.0.13 + '@golar/vue': 0.0.13 + peerDependenciesMeta: + '@golar/astro': + optional: true + '@golar/ember': + optional: true + '@golar/svelte': + optional: true + '@golar/vue': + optional: true + gopd@1.2.0: resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==} engines: {node: '>= 0.4'} @@ -6990,6 +7074,40 @@ snapshots: '@fastify/busboy@3.1.1': optional: true + '@golar/darwin-arm64@0.0.13': + optional: true + + '@golar/darwin-x64@0.0.13': + optional: true + + '@golar/linux-arm64@0.0.13': + optional: true + + '@golar/linux-x64@0.0.13': + optional: true + + '@golar/plugin@0.0.13': {} + + '@golar/util@0.0.13': {} + + '@golar/volar@0.0.13': + dependencies: + '@golar/plugin': 0.0.13 + '@volar/language-core': 2.4.28 + + '@golar/vue@0.0.13': + dependencies: + '@golar/util': 0.0.13 + '@golar/volar': 0.0.13 + '@vue/compiler-dom': 3.5.28 + '@vue/language-core': 3.2.2 + + '@golar/win32-arm64@0.0.13': + optional: true + + '@golar/win32-x64@0.0.13': + optional: true + '@headlessui/vue@1.7.23(vue@3.5.27(typescript@5.8.3))': dependencies: '@tanstack/vue-virtual': 3.13.12(vue@3.5.27(typescript@5.8.3)) @@ -8765,10 +8883,22 @@ snapshots: dependencies: '@volar/source-map': 2.4.23 + '@volar/language-core@2.4.27': + dependencies: + '@volar/source-map': 2.4.27 + + '@volar/language-core@2.4.28': + dependencies: + '@volar/source-map': 2.4.28 + '@volar/source-map@2.4.20': {} '@volar/source-map@2.4.23': {} + '@volar/source-map@2.4.27': {} + + '@volar/source-map@2.4.28': {} + '@volar/typescript@2.4.20': dependencies: '@volar/language-core': 2.4.20 @@ -8943,6 +9073,16 @@ snapshots: optionalDependencies: typescript: 5.8.3 + '@vue/language-core@3.2.2': + dependencies: + '@volar/language-core': 2.4.27 + '@vue/compiler-dom': 3.5.28 + '@vue/shared': 3.5.28 + alien-signals: 3.1.0 + muggle-string: 0.4.1 + path-browserify: 1.0.1 + picomatch: 4.0.3 + '@vue/reactivity@3.5.27': dependencies: '@vue/shared': 3.5.27 @@ -10367,6 +10507,16 @@ snapshots: slash: 5.1.0 unicorn-magic: 0.3.0 + golar@0.0.13(@golar/vue@0.0.13): + optionalDependencies: + '@golar/darwin-arm64': 0.0.13 + '@golar/darwin-x64': 0.0.13 + '@golar/linux-arm64': 0.0.13 + '@golar/linux-x64': 0.0.13 + '@golar/vue': 0.0.13 + '@golar/win32-arm64': 0.0.13 + '@golar/win32-x64': 0.0.13 + gopd@1.2.0: optional: true diff --git a/server/pnpm-workspace.yaml b/server/pnpm-workspace.yaml index f97bce83..9f2dfb72 100644 --- a/server/pnpm-workspace.yaml +++ b/server/pnpm-workspace.yaml @@ -1,4 +1,5 @@ onlyBuiltDependencies: + - "@bufbuild/buf" - "@parcel/watcher" - "@prisma/client" - "@prisma/engines" @@ -9,7 +10,4 @@ onlyBuiltDependencies: - sharp - unrs-resolver -# overrides: -# droplet: link:../../.local/share/pnpm/global/5/node_modules/@drop-oss/droplet - shamefullyHoist: true diff --git a/server/prisma/migrations/20260224145112_add_non_null_default_to_carousel_object_ids/migration.sql b/server/prisma/migrations/20260224145112_add_non_null_default_to_carousel_object_ids/migration.sql new file mode 100644 index 00000000..d86d3064 --- /dev/null +++ b/server/prisma/migrations/20260224145112_add_non_null_default_to_carousel_object_ids/migration.sql @@ -0,0 +1,15 @@ +-- DropIndex +DROP INDEX "Game_mName_idx"; + +-- DropIndex +DROP INDEX "GameTag_name_idx"; + +-- AlterTable +ALTER TABLE "Game" ALTER COLUMN "mImageCarouselObjectIds" SET DEFAULT ARRAY[]::TEXT[]; +UPDATE "Game" SET "mImageCarouselObjectIds" = '{}' WHERE "mImageCarouselObjectIds" IS NULL; + +-- CreateIndex +CREATE INDEX "Game_mName_idx" ON "Game" USING GIST ("mName" gist_trgm_ops(siglen=32)); + +-- CreateIndex +CREATE INDEX "GameTag_name_idx" ON "GameTag" USING GIST ("name" gist_trgm_ops(siglen=32)); diff --git a/server/prisma/models/content.prisma b/server/prisma/models/content.prisma index 1b2be612..25e900da 100644 --- a/server/prisma/models/content.prisma +++ b/server/prisma/models/content.prisma @@ -38,7 +38,7 @@ model Game { mIconObjectId String // linked to objects in s3 mBannerObjectId String // linked to objects in s3 mCoverObjectId String - mImageCarouselObjectIds String[] // linked to below array + mImageCarouselObjectIds String[] @default([]) // linked to below array mImageLibraryObjectIds String[] // linked to objects in s3 versions GameVersion[] diff --git a/server/server/api/v1/admin/game/index.get.ts b/server/server/api/v1/admin/game/index.get.ts deleted file mode 100644 index c7ca363f..00000000 --- a/server/server/api/v1/admin/game/index.get.ts +++ /dev/null @@ -1,16 +0,0 @@ -import aclManager from "~/server/internal/acls"; -import prisma from "~/server/internal/db/database"; - -export default defineEventHandler(async (h3) => { - const allowed = await aclManager.allowSystemACL(h3, ["game:read"]); - if (!allowed) throw createError({ statusCode: 403 }); - - return await prisma.game.findMany({ - select: { - id: true, - mName: true, - mShortDescription: true, - mIconObjectId: true, - }, - }); -}); diff --git a/server/server/api/v1/admin/import/massversion/index.get.ts b/server/server/api/v1/admin/import/massversion/index.get.ts new file mode 100644 index 00000000..3bfb286d --- /dev/null +++ b/server/server/api/v1/admin/import/massversion/index.get.ts @@ -0,0 +1,54 @@ +import aclManager from "~/server/internal/acls"; +import prisma from "~/server/internal/db/database"; +import { libraryManager } from "~/server/internal/library"; + +export default defineEventHandler(async (h3) => { + const allowed = await aclManager.allowSystemACL(h3, ["import:version:read"]); + if (!allowed) throw createError({ statusCode: 403 }); + + const games = await prisma.game.findMany({ + select: { + id: true, + mName: true, + mIconObjectId: true, + versions: { + select: { + versionPath: true, + }, + }, + unimportedGameVersions: { + select: { + id: true, + versionName: true, + }, + }, + libraryId: true, + libraryPath: true, + }, + }); + + const unimportedVersions = await Promise.all( + games.map(async (v) => ({ + id: v.id, + name: v.mName, + icon: v.mIconObjectId, + versions: await libraryManager.fetchUnimportedGameVersions( + v.libraryId, + v.libraryPath, + { + gameId: v.id, + versions: v.versions + .map((v) => v.versionPath) + .filter((v) => v !== null), + depotVersions: v.unimportedGameVersions, + }, + ), + })), + ); + + const onlyUnimported = unimportedVersions.filter( + (v) => v.versions && v.versions.length > 0, + ); + + return onlyUnimported; +}); diff --git a/server/server/api/v1/admin/import/massversion/index.post.ts b/server/server/api/v1/admin/import/massversion/index.post.ts new file mode 100644 index 00000000..99a0b7d3 --- /dev/null +++ b/server/server/api/v1/admin/import/massversion/index.post.ts @@ -0,0 +1,115 @@ +import { type } from "arktype"; +import { readDropValidatedBody, throwingArktype } from "~/server/arktype"; +import { aclManager } from "~/server/internal/acls"; +import { libraryManager } from "~/server/internal/library"; +import { taskHandler, wrapTaskContext } from "~/server/internal/tasks"; +import type { Platform } from "~/prisma/client/client"; + +const MassImport = type({ + versions: type({ + id: "string", + version: type({ + type: "'depot' | 'local'", + identifier: "string", + name: "string", + }), + displayName: "string?", + setupMode: "boolean = false", + }).array(), +}).configure(throwingArktype); + +export default defineEventHandler(async (h3) => { + const allowed = await aclManager.allowSystemACL(h3, ["import:version:new"]); + if (!allowed) throw createError({ statusCode: 403 }); + + const body = await readDropValidatedBody(h3, MassImport); + + const taskId = await taskHandler.create({ + key: "mass-import", + taskGroup: "import:version", + acls: ["system:import:version:read"], + name: `Mass-importing for ${body.versions.length} versions`, + async run({ progress, logger, addAction }) { + for ( + let versionIndex = 0; + versionIndex < body.versions.length; + versionIndex++ + ) { + const version = body.versions[versionIndex]; + const preload = await libraryManager.fetchUnimportedVersionInformation( + version.id, + version.version, + ); + if (!preload) { + logger.warn( + `failed to fetch preload information for: ${version.version.name} (${version.version.type})`, + ); + continue; + } + const chosenPreload = preload.at(0); + if (!chosenPreload) { + logger.warn( + `failed to find preload information for: ${version.version.name} (${version.version.type}), there were no auto-discovered executables`, + ); + continue; + } + + const launches: Array<{ + platform: Platform; + launch: string; + name: string; + }> = []; + const setups: Array<{ platform: Platform; launch: string }> = []; + + if (version.setupMode) { + setups.push({ + platform: chosenPreload.platform, + launch: chosenPreload.filename, + }); + } else { + launches.push({ + platform: chosenPreload.platform, + launch: chosenPreload.filename, + name: "Play", + }); + } + + logger.info(`importing ${version.version.name}`); + const min = versionIndex / body.versions.length; + const max = (versionIndex + 1) / body.versions.length; + + await libraryManager.importVersion( + version.id, + version.version, + { + id: version.id, + version: version.version, + launches, + setups, + onlySetup: version.setupMode, + delta: false, + requiredContent: [], + }, + wrapTaskContext( + { + logger, + progress, + addAction, + }, + { + min: min * 100, + max: max * 100, + prefix: `${version.version.name}`, + }, + ), + ); + + logger.info(`finished import for ${version.version.name}`); + + progress(max * 100); + } + }, + }); + + return { taskId }; +}); diff --git a/server/server/api/v1/admin/import/version/index.post.ts b/server/server/api/v1/admin/import/version/index.post.ts index 6f638256..9cae59cd 100644 --- a/server/server/api/v1/admin/import/version/index.post.ts +++ b/server/server/api/v1/admin/import/version/index.post.ts @@ -2,7 +2,6 @@ import { type } from "arktype"; import { Platform } from "~/prisma/client/enums"; import { readDropValidatedBody, throwingArktype } from "~/server/arktype"; import aclManager from "~/server/internal/acls"; -import prisma from "~/server/internal/db/database"; import libraryManager from "~/server/internal/library"; export const ImportVersion = type({ @@ -42,45 +41,6 @@ export default defineEventHandler(async (h3) => { const body = await readDropValidatedBody(h3, ImportVersion); - if (body.delta) { - for (const platformObject of [...body.launches, ...body.setups].filter( - (v, i, a) => a.findIndex((k) => k.platform === v.platform) == i, - )) { - const validOverlayVersions = await prisma.gameVersion.count({ - where: { - gameId: body.id, - delta: false, - OR: [ - { launches: { some: { platform: platformObject.platform } } }, - { - setups: { some: { platform: platformObject.platform } }, - }, - ], - }, - }); - if (validOverlayVersions == 0) - throw createError({ - statusCode: 400, - statusMessage: `Update mode requires a pre-existing version for platform: ${platformObject.platform}`, - }); - } - } - - if (body.onlySetup) { - if (body.setups.length == 0) - throw createError({ - statusCode: 400, - statusMessage: 'Setup required in "setup mode".', - }); - } else { - if (body.launches.length == 0) - throw createError({ - statusCode: 400, - statusMessage: "Launch executable is required.", - }); - } - - // startup & delta require more complex checking logic const taskId = await libraryManager.importVersion( body.id, body.version, diff --git a/server/server/api/v1/admin/library/index.get.ts b/server/server/api/v1/admin/library/index.get.ts index 5df358be..e43b158b 100644 --- a/server/server/api/v1/admin/library/index.get.ts +++ b/server/server/api/v1/admin/library/index.get.ts @@ -1,15 +1,128 @@ +import { ArkErrors, type } from "arktype"; +import type { SerializeObject } from "nitropack"; +import type { Prisma } from "~/prisma/client/client"; import aclManager from "~/server/internal/acls"; +import prisma from "~/server/internal/db/database"; import libraryManager from "~/server/internal/library"; +import deepmerge from "deepmerge"; + +const Query = type({ + query: "string?", + skip: "string.numeric.parse?", + limit: "string.numeric.parse?", + + sort: "'default' | 'newest' | 'recent' | 'name' = 'default'", + order: "'asc' | 'desc' = 'desc'", + + "filters?": type("string").pipe((s) => s.split(",")), +}); + +type FetchArg = Parameters[0]; + +export type AdminLibraryGame = SerializeObject< + Awaited>[number] +>; export default defineEventHandler(async (h3) => { const allowed = await aclManager.allowSystemACL(h3, ["library:read"]); if (!allowed) throw createError({ statusCode: 403 }); - const unimportedGames = await libraryManager.fetchUnimportedGames(); - const games = await libraryManager.fetchGamesWithStatus(); - const libraries = await libraryManager.fetchLibraries(); + const query = Query(getQuery(h3)); + if (query instanceof ArkErrors) + throw createError({ statusCode: 400, message: query.summary }); - // Fetch other library data here + const skip = query.skip + ? ({ + skip: query.skip, + } satisfies FetchArg) + : undefined; - return { unimportedGames, games, hasLibraries: libraries.length > 0 }; + const limit = Math.min(query.limit ?? 24, 50); + + const sort: Prisma.GameOrderByWithRelationInput = {}; + switch (query.sort) { + case "default": + case "newest": + sort.mReleased = query.order; + break; + case "recent": + sort.created = query.order; + break; + case "name": + sort.mName = query.order; + break; + } + + const rawFilters: Array = []; + if (query.filters && query.filters.length > 0) { + const filterSet = new Set(query.filters); + if (filterSet.has("version.none")) { + rawFilters.push({ + where: { + versions: { + none: {}, + }, + }, + }); + } + + if (filterSet.has("metadata.featured")) { + rawFilters.push({ + where: { + featured: true, + }, + }); + } + + if (filterSet.has("metadata.noCarousel")) { + rawFilters.push({ + where: { + OR: [ + { + mImageCarouselObjectIds: { + isEmpty: true, + }, + }, + ], + }, + }); + } + + if (filterSet.has("metadata.emptyDescription")) { + rawFilters.push({ + where: { + mDescription: "", + }, + }); + } + } + + if (query.query) { + rawFilters.push({ + where: { + mName: { + contains: query.query, + mode: "insensitive", + }, + }, + }); + } + + const filters = + rawFilters.length > 0 + ? rawFilters.reduce((a, b) => deepmerge(a, b)) + : undefined; + + const results = await libraryManager.fetchGamesWithStatus({ + ...skip, + take: limit, + orderBy: sort, + ...filters, + }); + + // Safety: the type is defined as a union between the where and count args + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const count = await prisma.game.count({ ...(filters as any) }); + + return { results, count }; }); diff --git a/server/server/api/v1/admin/library/libraries.get.ts b/server/server/api/v1/admin/library/libraries.get.ts new file mode 100644 index 00000000..3eda8061 --- /dev/null +++ b/server/server/api/v1/admin/library/libraries.get.ts @@ -0,0 +1,12 @@ +import aclManager from "~/server/internal/acls"; +import libraryManager from "~/server/internal/library"; + +export default defineEventHandler(async (h3) => { + const allowed = await aclManager.allowSystemACL(h3, ["library:read"]); + if (!allowed) throw createError({ statusCode: 403 }); + + const unimportedGames = await libraryManager.fetchUnimportedGames(); + const libraries = await libraryManager.fetchLibraries(); + + return { unimportedGames, hasLibraries: libraries.length > 0 }; +}); diff --git a/server/server/api/v1/admin/task/index.get.ts b/server/server/api/v1/admin/task/index.get.ts index 690dce82..18d2b418 100644 --- a/server/server/api/v1/admin/task/index.get.ts +++ b/server/server/api/v1/admin/task/index.get.ts @@ -1,6 +1,5 @@ import aclManager from "~/server/internal/acls"; import prisma from "~/server/internal/db/database"; -import type { TaskMessage } from "~/server/internal/tasks"; import taskHandler from "~/server/internal/tasks"; export default defineEventHandler(async (h3) => { @@ -14,7 +13,7 @@ export default defineEventHandler(async (h3) => { }); const runningTasks = (await taskHandler.runningTasks()).map((e) => e.id); - const historicalTasks = (await prisma.task.findMany({ + const historicalTasks = await prisma.task.findMany({ where: { OR: [ { @@ -28,8 +27,15 @@ export default defineEventHandler(async (h3) => { orderBy: { ended: "desc", }, - take: 10, - })) as Array; + select: { + id: true, + name: true, + actions: true, + error: true, + success: true, + }, + take: 32, + }); const dailyTasks = await taskHandler.dailyTasks(); const weeklyTasks = await taskHandler.weeklyTasks(); diff --git a/server/server/api/v1/client/game/[id]/versions.get.ts b/server/server/api/v1/client/game/[id]/versions.get.ts index b5d6a43c..c59d4320 100644 --- a/server/server/api/v1/client/game/[id]/versions.get.ts +++ b/server/server/api/v1/client/game/[id]/versions.get.ts @@ -1,3 +1,4 @@ +import { ArkErrors, type } from "arktype"; import type { Platform } from "~/prisma/client/enums"; import { defineClientEventHandler } from "~/server/internal/clients/event-handler"; import prisma from "~/server/internal/db/database"; @@ -21,6 +22,10 @@ type VersionDownloadOption = { }>; }; +const Query = type({ + previous: "string?", +}); + export default defineClientEventHandler(async (h3) => { const id = getRouterParam(h3, "id")!; if (!id) @@ -29,6 +34,10 @@ export default defineClientEventHandler(async (h3) => { statusMessage: "No ID in router params", }); + const query = Query(getQuery(h3)); + if (query instanceof ArkErrors) + throw createError({ statusCode: 400, message: query.summary }); + const rawVersions = await prisma.gameVersion.findMany({ where: { gameId: id, @@ -93,7 +102,10 @@ export default defineClientEventHandler(async (h3) => { } } - const size = await gameSizeManager.getVersionSize(v.versionId); + const size = await gameSizeManager.getVersionSize( + v.versionId, + query.previous, + ); return platformOptions .entries() diff --git a/server/server/api/v1/client/game/manifest.get.ts b/server/server/api/v1/client/game/manifest.get.ts index 44208a40..06018435 100644 --- a/server/server/api/v1/client/game/manifest.get.ts +++ b/server/server/api/v1/client/game/manifest.get.ts @@ -1,15 +1,22 @@ +import { ArkErrors, type } from "arktype"; import { defineClientEventHandler } from "~/server/internal/clients/event-handler"; import { createDownloadManifestDetails } from "~/server/internal/library/manifest/index"; -export default defineClientEventHandler(async (h3) => { - const query = getQuery(h3); - const version = query.version?.toString(); - if (!version) - throw createError({ - statusCode: 400, - statusMessage: "Missing version ID in query", - }); +const Query = type({ + version: "string", + previous: "string?", + refresh: "string?", +}); - const result = await createDownloadManifestDetails(version); +export default defineClientEventHandler(async (h3) => { + const query = Query(getQuery(h3)); + if (query instanceof ArkErrors) + throw createError({ statusCode: 400, message: query.summary }); + + const result = await createDownloadManifestDetails( + query.version, + query.previous, + query.refresh == "true", + ); return result; }); diff --git a/server/server/internal/gamesize/index.ts b/server/server/internal/gamesize/index.ts index ce4bd6e6..2a56489f 100644 --- a/server/server/internal/gamesize/index.ts +++ b/server/server/internal/gamesize/index.ts @@ -21,23 +21,33 @@ class GameSizeManager { private gameBreakdownCache = cacheHandler.createCache("gameBreakdown"); + private gameVersionSizeCacheKey(versionId: string, previousId?: string) { + return `${versionId}${previousId ? `-from-${previousId}` : ""}`; + } + /*** * Gets the size of the game to the user: * - installSize: size on disk after install * - downloadSize: how many bytes are downloaded (but not necessarily stored) */ - async getVersionSize(versionId: string): Promise { - if (await this.gameVersionsSizesCache.has(versionId)) - return await this.gameVersionsSizesCache.get(versionId); + async getVersionSize( + versionId: string, + previousId?: string, + ): Promise { + const key = this.gameVersionSizeCacheKey(versionId, previousId); + if (await this.gameVersionsSizesCache.has(key)) + return await this.gameVersionsSizesCache.get(key); try { - const { downloadSize, installSize } = - await createDownloadManifestDetails(versionId); + const { downloadSize, installSize } = await createDownloadManifestDetails( + versionId, + previousId, + ); const result = { downloadSize, installSize, versionId, } satisfies GameVersionSize; - await this.gameVersionsSizesCache.set(versionId, result); + await this.gameVersionsSizesCache.set(key, result); return result; } catch { return null; diff --git a/server/server/internal/library/index.ts b/server/server/internal/library/index.ts index 057f5ad6..3aafa6ef 100644 --- a/server/server/internal/library/index.ts +++ b/server/server/internal/library/index.ts @@ -8,6 +8,7 @@ import path from "path"; import prisma from "../db/database"; import { fuzzy } from "fast-fuzzy"; +import type { TaskRunContext } from "../tasks"; import taskHandler from "../tasks"; import notificationSystem from "../notifications"; import { GameNotFoundError, type LibraryProvider } from "./provider"; @@ -20,6 +21,7 @@ import type { ImportVersion } from "~/server/api/v1/admin/import/version/index.p import { GameType, type Platform } from "~/prisma/client/enums"; import { castManifest } from "./manifest/utils"; import { Shescape } from "shescape"; +import type { Prisma } from "~/prisma/client/client"; export function createGameImportTaskId(libraryId: string, libraryPath: string) { return createHash("md5") @@ -125,42 +127,33 @@ class LibraryManager { async fetchUnimportedGameVersions( libraryId: string, libraryPath: string, + noFetchParams?: { + gameId: string; + versions: string[]; + depotVersions: { id: string; versionName: string }[]; + }, ): Promise { const provider = this.libraries.get(libraryId); if (!provider) return undefined; - const game = await prisma.game.findUnique({ - where: { - libraryKey: { - libraryId, - libraryPath, + let params = noFetchParams; + if (!params) { + const game = await prisma.game.findUnique({ + where: { + libraryKey: { + libraryId, + libraryPath, + }, }, - }, - select: { - id: true, - versions: true, - }, - }); - if (!game) return undefined; - - try { - const versions = await provider.listVersions( - libraryPath, - game.versions.map((v) => v.versionPath).filter((v) => v !== null), - ); - const unimportedVersions = versions - .filter( - (e) => - game.versions.findIndex((v) => v.versionPath == e) == -1 && - !taskHandler.hasTaskKey(createVersionImportTaskKey(game.id, e)), - ) - .map( - (v) => - ({ - type: "local", - name: v, - identifier: v, - }) satisfies UnimportedVersionInformation, - ); + select: { + id: true, + versions: { + select: { + versionPath: true, + }, + }, + }, + }); + if (!game) return undefined; const depotVersions = await prisma.unimportedGameVersion.findMany({ where: { gameId: game.id, @@ -170,7 +163,38 @@ class LibraryManager { id: true, }, }); - const mappedDepotVersions = depotVersions.map( + + params = { + gameId: game.id, + versions: game.versions + .map((v) => v.versionPath) + .filter((v) => v !== null), + depotVersions: depotVersions, + }; + } + + try { + const versions = await provider.listVersions( + libraryPath, + params.versions, + ); + const unimportedVersions = versions + .filter( + (e) => + params.versions.findIndex((v) => v == e) == -1 && + !taskHandler.hasTaskKey( + createVersionImportTaskKey(params.gameId, e), + ), + ) + .map( + (v) => + ({ + type: "local", + name: v, + identifier: v, + }) satisfies UnimportedVersionInformation, + ); + const mappedDepotVersions = params.depotVersions.map( (v) => ({ type: "depot", @@ -188,29 +212,37 @@ class LibraryManager { } } - async fetchGamesWithStatus() { + async fetchGamesWithStatus( + where: Partial>, + ) { const games = await prisma.game.findMany({ + ...where, include: { library: true, versions: true, - }, - orderBy: { - mName: "asc", + unimportedGameVersions: true, }, }); return await Promise.all( games.map(async (e) => { - const versions = await this.fetchUnimportedGameVersions( + const unimportedVersions = await this.fetchUnimportedGameVersions( e.libraryId ?? "", e.libraryPath, + { + gameId: e.id, + versions: e.versions + .map((v) => v.versionPath) + .filter((v) => v !== null), + depotVersions: e.unimportedGameVersions, + }, ); return { game: e, - status: versions + status: unimportedVersions ? { noVersions: e.versions.length == 0, - unimportedVersions: versions, + unimportedVersions: unimportedVersions, } : ("offline" as const), }; @@ -375,9 +407,51 @@ class LibraryManager { gameId: string, version: UnimportedVersionInformation, metadata: typeof ImportVersion.infer, + parentTask?: TaskRunContext, ) { const taskKey = createVersionImportTaskKey(gameId, version.identifier); + if (metadata.delta) { + for (const platformObject of [ + ...metadata.launches, + ...metadata.setups, + ].filter( + (v, i, a) => a.findIndex((k) => k.platform === v.platform) == i, + )) { + const validOverlayVersions = await prisma.gameVersion.count({ + where: { + gameId: metadata.id, + delta: false, + OR: [ + { launches: { some: { platform: platformObject.platform } } }, + { + setups: { some: { platform: platformObject.platform } }, + }, + ], + }, + }); + if (validOverlayVersions == 0) + throw createError({ + statusCode: 400, + message: `Update mode requires a pre-existing version for platform: ${platformObject.platform}`, + }); + } + } + + if (metadata.onlySetup) { + if (metadata.setups.length == 0) + throw createError({ + statusCode: 400, + message: 'Setup required in "setup mode".', + }); + } else { + if (metadata.launches.length == 0) + throw createError({ + statusCode: 400, + message: "Launch executable is required.", + }); + } + const game = await prisma.game.findUnique({ where: { id: gameId }, select: { mName: true, libraryId: true, libraryPath: true, type: true }, @@ -400,124 +474,134 @@ class LibraryManager { }) : undefined; - return await taskHandler.create({ - key: taskKey, - taskGroup: "import:game", - name: `Importing version ${version.name} for ${game.mName}`, - acls: ["system:import:version:read"], - async run({ progress, logger }) { - let versionPath: string | null = null; - let manifest; - let fileList; + return await taskHandler.create( + { + key: taskKey, + taskGroup: "import:version", + name: `Importing version ${version.name} for ${game.mName}`, + acls: ["system:import:version:read"], + async run({ progress, logger }) { + let versionPath: string | null = null; + let manifest; + let fileList; - if (version.type === "local") { - versionPath = version.identifier; - // First, create the manifest via droplet. - // This takes up 90% of our progress, so we wrap it in a *0.9 + if (version.type === "local") { + versionPath = version.identifier; + // First, create the manifest via droplet. + // This takes up 90% of our progress, so we wrap it in a *0.9 - manifest = await library.generateDropletManifest( - game.libraryPath, - versionPath, - (value) => { - progress(value * 0.9); - }, - (value) => { - logger.info(value); - }, - ); - fileList = await library.versionReaddir( - game.libraryPath, - versionPath, - ); - logger.info("Created manifest successfully!"); - } else if (version.type === "depot" && unimportedVersion) { - manifest = castManifest(unimportedVersion.manifest); - fileList = unimportedVersion.fileList; - progress(90); - } else { - throw "Could not find or create manifest for this version."; - } - - const currentIndex = await prisma.gameVersion.count({ - where: { gameId: gameId }, - }); - - // Then, create the database object - const newVersion = await prisma.gameVersion.create({ - data: { - game: { - connect: { - id: gameId, + manifest = await library.generateDropletManifest( + game.libraryPath, + versionPath, + (value) => { + progress(value * 0.9); }, - }, - - displayName: metadata.displayName ?? null, - - versionPath, - dropletManifest: manifest, - fileList, - versionIndex: currentIndex, - delta: metadata.delta, - - onlySetup: metadata.onlySetup, - setups: { - createMany: { - data: metadata.setups.map((v) => ({ - command: v.launch, - platform: v.platform, - })), + (value) => { + logger.info(value); }, + ); + fileList = await library.versionReaddir( + game.libraryPath, + versionPath, + ); + logger.info("Created manifest successfully!"); + } else if (version.type === "depot" && unimportedVersion) { + manifest = castManifest(unimportedVersion.manifest); + fileList = unimportedVersion.fileList; + progress(90); + } else { + throw "Could not find or create manifest for this version."; + } + + const largestIndex = await prisma.gameVersion.findFirst({ + where: { gameId: gameId }, + orderBy: { + versionIndex: "desc", }, - - launches: { - createMany: !metadata.onlySetup - ? { - data: metadata.launches.map((v) => ({ - name: v.name, - command: v.launch, - platform: v.platform, - ...(v.emulatorId && game.type === "Game" - ? { - emulatorId: v.emulatorId, - } - : undefined), - emulatorSuggestions: - game.type === "Emulator" ? (v.suggestions ?? []) : [], - })), - } - : { data: [] }, - }, - }, - }); - logger.info("Successfully created version!"); - - notificationSystem.systemPush({ - nonce: `version-create-${gameId}-${version}`, - title: `'${game.mName}' ('${version.name}') finished importing.`, - description: `Drop finished importing version ${version.name} for ${game.mName}.`, - actions: [`View|/admin/library/${gameId}`], - acls: ["system:import:version:read"], - }); - - // Ensure cache is filled (also pre-caches the manifest) - try { - await gameSizeManager.getVersionSize(newVersion.versionId); - } catch (e) { - logger.warn(`Failed to pre-cache game size and manifest: ${e}`); - } - - if (version.type === "depot") { - // SAFETY: we can only reach this if the type is depot and identifier is valid - // eslint-disable-next-line drop/no-prisma-delete - await prisma.unimportedGameVersion.delete({ - where: { - id: version.identifier, + select: { + versionIndex: true, }, }); - } - progress(100); + const currentIndex = largestIndex ? largestIndex.versionIndex + 1 : 0; + + // Then, create the database object + const newVersion = await prisma.gameVersion.create({ + data: { + game: { + connect: { + id: gameId, + }, + }, + + displayName: metadata.displayName ?? null, + + versionPath, + dropletManifest: manifest, + fileList, + versionIndex: currentIndex, + delta: metadata.delta, + + onlySetup: metadata.onlySetup, + setups: { + createMany: { + data: metadata.setups.map((v) => ({ + command: v.launch, + platform: v.platform, + })), + }, + }, + + launches: { + createMany: !metadata.onlySetup + ? { + data: metadata.launches.map((v) => ({ + name: v.name, + command: v.launch, + platform: v.platform, + ...(v.emulatorId && game.type === "Game" + ? { + emulatorId: v.emulatorId, + } + : undefined), + emulatorSuggestions: + game.type === "Emulator" ? (v.suggestions ?? []) : [], + })), + } + : { data: [] }, + }, + }, + }); + logger.info("Successfully created version!"); + + notificationSystem.systemPush({ + nonce: `version-create-${gameId}-${version}`, + title: `'${game.mName}' ('${version.name}') finished importing.`, + description: `Drop finished importing version ${version.name} for ${game.mName}.`, + actions: [`View|/admin/library/${gameId}`], + acls: ["system:import:version:read"], + }); + + // Ensure cache is filled (also pre-caches the manifest) + try { + await gameSizeManager.getVersionSize(newVersion.versionId); + } catch (e) { + logger.warn(`Failed to pre-cache game size and manifest: ${e}`); + } + + if (version.type === "depot") { + // SAFETY: we can only reach this if the type is depot and identifier is valid + // eslint-disable-next-line drop/no-prisma-delete + await prisma.unimportedGameVersion.delete({ + where: { + id: version.identifier, + }, + }); + } + progress(100); + }, }, - }); + parentTask, + ); } async peekFile( diff --git a/server/server/internal/library/manifest/index.ts b/server/server/internal/library/manifest/index.ts index c606eae2..91fdb187 100644 --- a/server/server/internal/library/manifest/index.ts +++ b/server/server/internal/library/manifest/index.ts @@ -30,10 +30,12 @@ const manifestCache = */ export async function createDownloadManifestDetails( versionId: string, + previous?: string, refresh = false, ): Promise { - if ((await manifestCache.has(versionId)) && !refresh) - return (await manifestCache.get(versionId))!; + const manifestKey = `${versionId}${previous ? `-from-${previous}` : ""}`; + if ((await manifestCache.has(manifestKey)) && !refresh) + return (await manifestCache.get(manifestKey))!; const mainVersion = await prisma.gameVersion.findUnique({ where: { versionId }, select: { @@ -94,6 +96,10 @@ export async function createDownloadManifestDetails( let installSize = 0; let downloadSize = 0; + const existingChunks = previous + ? await createDownloadManifestDetails(previous) + : undefined; + // Now that we have our file list, filter the manifests const manifests = new Map(); for (const version of versionOrder) { @@ -105,9 +111,15 @@ export async function createDownloadManifestDetails( const fileNames = Object.fromEntries(files); const manifest = castManifest(version.dropletManifest); const filteredChunks = Object.fromEntries( - Object.entries(manifest.chunks).filter(([, chunkData]) => { + Object.entries(manifest.chunks).filter(([_, chunkData]) => { + //if(existingChunks && existingChunks.manifests[version.versionId]?.chunks?.[chunkId]) return false; let flag = false; chunkData.files.forEach((fileEntry) => { + if ( + existingChunks && + existingChunks.fileList[fileEntry.filename] == version.versionId + ) + return; if (fileNames[fileEntry.filename]) { flag = true; installSize += fileEntry.length; @@ -134,7 +146,7 @@ export async function createDownloadManifestDetails( installSize, downloadSize, }; - await manifestCache.set(versionId, result); + await manifestCache.set(manifestKey, result); return result; } diff --git a/server/server/internal/services/index.ts b/server/server/internal/services/index.ts index c019fe67..822a7199 100644 --- a/server/server/internal/services/index.ts +++ b/server/server/internal/services/index.ts @@ -107,9 +107,12 @@ export class Service { if (this.spun) this.launch(); }); - serviceProcess.stdout?.on("data", (data) => - this.logger.info(data.toString().trim()), - ); + serviceProcess.stdout?.on("data", (data) => { + const lines = data.toString().trim().split("\n"); + for (const line of lines) { + this.logger.info(line); + } + }); serviceProcess.stderr?.on("data", (data) => this.logger.error(data.toString().trim()), diff --git a/server/server/internal/tasks/group.ts b/server/server/internal/tasks/group.ts index f914272a..4ddef804 100644 --- a/server/server/internal/tasks/group.ts +++ b/server/server/internal/tasks/group.ts @@ -14,6 +14,9 @@ export const taskGroups = { "import:game": { concurrency: true, }, + "import:version": { + concurrency: true, + }, debug: { concurrency: true, }, diff --git a/server/server/internal/tasks/index.ts b/server/server/internal/tasks/index.ts index 189dac09..8a47483f 100644 --- a/server/server/internal/tasks/index.ts +++ b/server/server/internal/tasks/index.ts @@ -8,7 +8,7 @@ import checkUpdate from "./registry/update"; import cleanupObjects from "./registry/objects"; import { taskGroups, type TaskGroup } from "./group"; import prisma from "../db/database"; -import { type } from "arktype"; +import { ArkErrors, type } from "arktype"; import pino from "pino"; import { logger } from "~/server/internal/logging"; import { Writable } from "node:stream"; @@ -76,7 +76,7 @@ class TaskHandler { this.taskCreators.set(task.taskGroup, task.build); } - async create(iTask: Omit) { + async create(iTask: Omit, parentTask?: TaskRunContext) { const task: Task = { ...iTask, id: crypto.randomUUID() }; if (this.hasTaskID(task.id)) throw new Error("Task with ID already exists."); @@ -105,6 +105,7 @@ class TaskHandler { const updateAllClients = (reset = false) => new Promise((r) => { + //if (parentTask) return; // NO-OP if we're a child task if (updateCollectTimeout) { updateCollectResolves.push(r); return; @@ -148,7 +149,10 @@ class TaskHandler { write(chunk, encoding, callback) { try { // chunk is a stringified JSON log line - const logObj = JSON.parse(chunk.toString()); + const logObj = TaskLog(JSON.parse(chunk.toString())); + if (logObj instanceof ArkErrors) { + throw logObj; + } const taskEntry = taskPool.get(task.id); if (taskEntry) { taskEntry.log.push(JSON.stringify(logObj)); @@ -156,43 +160,44 @@ class TaskHandler { } } catch (e) { // fallback: ignore or log error - logger.error("Failed to parse log chunk", { - error: e, - chunk: chunk, - }); + logger.error(`Failed to parse log chunk: ${e}, ${chunk}`); } callback(); }, }); // Use pino with the custom stream - const taskLogger = pino( - { - // You can configure timestamp, level, etc. here - timestamp: pino.stdTimeFunctions.isoTime, - base: null, // Remove pid/hostname if not needed - formatters: { - level(label) { - return { - level: label, - }; + const taskLogger = + parentTask?.logger ?? + pino( + { + // You can configure timestamp, level, etc. here + timestamp: pino.stdTimeFunctions.isoTime, + base: null, // Remove pid/hostname if not needed + formatters: { + level(label) { + return { + level: label, + }; + }, }, }, - }, - logStream, - ); + logStream, + ); - const progress = (progress: number) => { - if (progress < 0 || progress > 100) { - logger.error("Progress must be between 0 and 100", { progress }); - return; - } - const taskEntry = this.taskPool.get(task.id); - if (!taskEntry) return; - taskEntry.progress = progress; - // log(`Progress: ${progress}%`); - updateAllClients(); - }; + const progress = + parentTask?.progress ?? + ((progress: number) => { + if (progress < 0 || progress > 100) { + logger.error("Progress must be between 0 and 100", { progress }); + return; + } + const taskEntry = this.taskPool.get(task.id); + if (!taskEntry) return; + taskEntry.progress = progress; + // log(`Progress: ${progress}%`); + updateAllClients(); + }); this.taskPool.set(task.id, { name: task.name, @@ -233,35 +238,37 @@ class TaskHandler { taskEntry.endTime = new Date().toISOString(); await updateAllClients(); - for (const clientId of taskEntry.clients.keys()) { - if (!this.clientRegistry.get(clientId)) continue; - this.disconnect(clientId, task.id); + if (!parentTask) { + for (const clientId of taskEntry.clients.keys()) { + if (!this.clientRegistry.get(clientId)) continue; + this.disconnect(clientId, task.id); + } + + await prisma.task.create({ + data: { + id: task.id, + taskGroup: taskEntry.taskGroup, + name: taskEntry.name, + + started: taskEntry.startTime, + ended: taskEntry.endTime, + + success: taskEntry.success, + progress: taskEntry.progress, + log: taskEntry.log, + + acls: taskEntry.acls, + actions: taskEntry.actions, + + ...(taskEntry.error ? { error: taskEntry.error } : undefined), + }, + }); } - - await prisma.task.create({ - data: { - id: task.id, - taskGroup: taskEntry.taskGroup, - name: taskEntry.name, - - started: taskEntry.startTime, - ended: taskEntry.endTime, - - success: taskEntry.success, - progress: taskEntry.progress, - log: taskEntry.log, - - acls: taskEntry.acls, - actions: taskEntry.actions, - - ...(taskEntry.error ? { error: taskEntry.error } : undefined), - }, - }); - this.taskPool.delete(task.id); }; - taskFunc(); + const fnPromise = taskFunc(); + if (parentTask) await fnPromise; return task.id; } @@ -511,9 +518,10 @@ interface DropTask { } export const TaskLog = type({ - timestamp: "string", - message: "string", + time: "string", + msg: "string", level: "string", + prefix: "string?", }); // /** diff --git a/server/torrential b/server/torrential index 50e54b6c..f469744e 160000 --- a/server/torrential +++ b/server/torrential @@ -1 +1 @@ -Subproject commit 50e54b6c60f76dd5e5121e601e06f3ebb633014b +Subproject commit f469744ebf37f625b106dbebfcb83d0ab836262b diff --git a/server/utils/parseTaskLog.ts b/server/utils/parseTaskLog.ts index 93f1e21f..7c02bf16 100644 --- a/server/utils/parseTaskLog.ts +++ b/server/utils/parseTaskLog.ts @@ -14,7 +14,7 @@ const labelNumberMap = { export function parseTaskLog( logStr?: string | undefined, ): typeof TaskLog.infer { - if (!logStr) return { message: "", timestamp: "", level: "" }; + if (!logStr) return { msg: "", time: "", level: "" }; const log = JSON.parse(logStr); if (typeof log.level === "number") { @@ -23,9 +23,5 @@ export function parseTaskLog( ] as string; } - return { - message: log.msg, - timestamp: log.time, - level: log.level, - }; + return log; }