jessikitty/drop
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

Depot API & v4 (#298)

Browse Source 
* feat: nginx + torrential basics & services system

* fix: lint + i18n

* fix: update torrential to remove openssl

* feat: add torrential to Docker build

* feat: move to self hosted runner

* fix: move off self-hosted runner

* fix: update nginx.conf

* feat: torrential cache invalidation

* fix: update torrential for cache invalidation

* feat: integrity check task

* fix: lint

* feat: move to version ids

* fix: client fixes and client-side checks

* feat: new depot apis and version id fixes

* feat: update torrential

* feat: droplet bump and remove unsafe update functions

* fix: lint

* feat: v4 featureset: emulators, multi-launch commands

* fix: lint

* fix: mobile ui for game editor

* feat: launch options

* fix: lint

* fix: remove axios, use $fetch

* feat: metadata and task api improvements

* feat: task actions

* fix: slight styling issue

* feat: fix style and lints

* feat: totp backend routes

* feat: oidc groups

* fix: update drop-base

* feat: creation of passkeys & totp

* feat: totp signin

* feat: webauthn mfa/signin

* feat: launch selecting ui

* fix: manually running tasks

* feat: update add company game modal to use new SelectorGame

* feat: executor selector

* fix(docker): update rust to rust nightly for torrential build (#305)

* feat: new version ui

* feat: move package lookup to build time to allow for deno dev

* fix: lint

* feat: localisation cleanup

* feat: apply localisation cleanup

* feat: potential i18n refactor logic

* feat: remove args from commands

* fix: lint

* fix: lockfile

---------

Co-authored-by: Aden Lindsay <140392385+AdenMGB@users.noreply.github.com>
This commit is contained in:
DecDuck
2026-01-13 15:32:39 +11:00
committed by GitHub
parent 8ef983304c
commit 63ac2b8ffc
190 changed files with 5848 additions and 2309 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/internal/metadata/giantbomb.ts
+11 -18
View File
@@ -10,10 +10,10 @@ import type {
CompanyMetadata,
GameMetadataRating,
} from "./types";
import axios, { type AxiosRequestConfig } from "axios";
import TurndownService from "turndown";
import { DateTime } from "luxon";
import type { TaskRunContext } from "../tasks";
import type { NitroFetchOptions, NitroFetchRequest } from "nitropack";
interface GiantBombResponseType<T> {
error: "OK" | string;
@@ -120,7 +120,7 @@ export class GiantBombProvider implements MetadataProvider {
resource: string,
url: string,
query: { [key: string]: string },
options?: AxiosRequestConfig,
options?: NitroFetchOptions<NitroFetchRequest, "post">,
) {
const queryString = new URLSearchParams({
...query,
@@ -130,13 +130,7 @@ export class GiantBombProvider implements MetadataProvider {
const finalURL = `https://www.giantbomb.com/api/${resource}/${url}?${queryString}`;
const overlay: AxiosRequestConfig = {
url: finalURL,
baseURL: "",
};
const response = await axios.request<GiantBombResponseType<T>>(
Object.assign({}, options, overlay),
);
const response = await $fetch<GiantBombResponseType<T>>(finalURL, options);
return response;
}
@@ -152,7 +146,7 @@ export class GiantBombProvider implements MetadataProvider {
query: query,
resources: ["game"].join(","),
});
const mapped = results.data.results.map((result) => {
const mapped = results.results.map((result) => {
const date =
(result.original_release_date
? DateTime.fromISO(result.original_release_date).year
@@ -172,13 +166,13 @@ export class GiantBombProvider implements MetadataProvider {
return mapped;
}
async fetchGame(
{ id, publisher, developer, createObject }: _FetchGameMetadataParams,
{ id, company, createObject }: _FetchGameMetadataParams,
context?: TaskRunContext,
): Promise<GameMetadata> {
context?.logger.info("Using GiantBomb provider");
const result = await this.request<GameResult>("game", id, {});
const gameData = result.data.results;
const gameData = result.results;
const longDescription = gameData.description
? this.turndown.turndown(gameData.description)
@@ -189,7 +183,7 @@ export class GiantBombProvider implements MetadataProvider {
for (const pub of gameData.publishers) {
context?.logger.info(`Importing publisher "${pub.name}"`);
const res = await publisher(pub.name);
const res = await company(pub.name);
if (res === undefined) {
context?.logger.warn(`Failed to import publisher "${pub.name}"`);
continue;
@@ -206,7 +200,7 @@ export class GiantBombProvider implements MetadataProvider {
for (const dev of gameData.developers) {
context?.logger.info(`Importing developer "${dev.name}"`);
const res = await developer(dev.name);
const res = await company(dev.name);
if (res === undefined) {
context?.logger.warn(`Failed to import developer "${dev.name}"`);
continue;
@@ -244,8 +238,8 @@ export class GiantBombProvider implements MetadataProvider {
metadataSource: MetadataSource.GiantBomb,
metadataId: reviewId,
mReviewCount: 1,
mReviewRating: review.data.results.score / 5,
mReviewHref: review.data.results.site_detail_url,
mReviewRating: review.results.score / 5,
mReviewHref: review.results.site_detail_url,
});
}
}
@@ -289,8 +283,7 @@ export class GiantBombProvider implements MetadataProvider {
// Find the right entry
const company =
results.data.results.find((e) => e.name == query) ??
results.data.results.at(0);
results.results.find((e) => e.name == query) ?? results.results.at(0);
if (!company) return undefined;
const longDescription = company.description
server/internal/metadata/igdb.ts
+27 -50
View File
@@ -9,12 +9,11 @@ import type {
_FetchCompanyMetadataParams,
CompanyMetadata,
} from "./types";
import type { AxiosRequestConfig } from "axios";
import axios from "axios";
import { DateTime } from "luxon";
import * as jdenticon from "jdenticon";
import type { TaskRunContext } from "../tasks";
import { logger } from "~/server/internal/logging";
import type { NitroFetchOptions, NitroFetchRequest } from "nitropack";
type IGDBID = number;
@@ -171,20 +170,16 @@ export class IGDBProvider implements MetadataProvider {
grant_type: "client_credentials",
});
const response = await axios.request<TwitchAuthResponse>({
url: `https://id.twitch.tv/oauth2/token?${params.toString()}`,
baseURL: "",
method: "POST",
});
const response = await $fetch<TwitchAuthResponse>(
`https://id.twitch.tv/oauth2/token?${params.toString()}`,
{
method: "POST",
},
);
if (response.status !== 200)
throw new Error(
`Error in IGDB \nStatus Code: ${response.status}\n${response.data}`,
);
this.accessToken = response.data.access_token;
this.accessToken = response.access_token;
this.accessTokenExpiry = DateTime.now().plus({
seconds: response.data.expires_in,
seconds: response.expires_in,
});
logger.info("IGDB done authorizing with twitch");
@@ -202,7 +197,7 @@ export class IGDBProvider implements MetadataProvider {
private async request<T extends object>(
resource: string,
body: string,
options?: AxiosRequestConfig,
options?: NitroFetchOptions<NitroFetchRequest, "post">,
) {
await this.refreshCredentials();
@@ -214,11 +209,10 @@ export class IGDBProvider implements MetadataProvider {
const finalURL = `https://api.igdb.com/v4/${resource}`;
const overlay: AxiosRequestConfig = {
url: finalURL,
const overlay: NitroFetchOptions<NitroFetchRequest, "post"> = {
baseURL: "",
method: "POST",
data: body,
body,
headers: {
Accept: "application/json",
"Client-ID": this.clientId,
@@ -226,24 +220,13 @@ export class IGDBProvider implements MetadataProvider {
"content-type": "text/plain",
},
};
const response = await axios.request<T[] | IGDBErrorResponse[]>(
const response = await $fetch<T[] | IGDBErrorResponse[]>(
finalURL,
Object.assign({}, options, overlay),
);
if (response.status !== 200) {
let cause = "";
response.data.forEach((item) => {
if ("cause" in item) cause = item.cause;
});
throw new Error(
`Error in igdb \nStatus Code: ${response.status} \nCause: ${cause}`,
);
}
// should not have an error object if the status code is 200
return <T[]>response.data;
return <T[]>response;
}
private async _getMediaInternal(
@@ -356,7 +339,7 @@ export class IGDBProvider implements MetadataProvider {
return results;
}
async fetchGame(
{ id, publisher, developer, createObject }: _FetchGameMetadataParams,
{ id, company, createObject }: _FetchGameMetadataParams,
context?: TaskRunContext,
): Promise<GameMetadata> {
const body = `where id = ${id}; fields *;`;
@@ -416,34 +399,28 @@ export class IGDBProvider implements MetadataProvider {
{ name: string } & IGDBItem
>("companies", `where id = ${foundInvolved.company}; fields name;`);
for (const company of findCompanyResponse) {
for (const companyData of findCompanyResponse) {
context?.logger.info(
`Found involved company "${company.name}" as: ${foundInvolved.developer ? "developer, " : ""}${foundInvolved.publisher ? "publisher" : ""}`,
);
const res = await company(companyData.name);
if (res === undefined) {
context?.logger.warn(
`Failed to import company "${companyData.name}"`,
);
continue;
}
// if company was a dev or publisher
// CANNOT use else since a company can be both
if (foundInvolved.developer) {
const res = await developer(company.name);
if (res === undefined) {
context?.logger.warn(
`Failed to import developer "${company.name}"`,
);
continue;
}
context?.logger.info(`Imported developer "${company.name}"`);
context?.logger.info(`Imported developer "${companyData.name}"`);
developers.push(res);
}
if (foundInvolved.publisher) {
const res = await publisher(company.name);
if (res === undefined) {
context?.logger.warn(
`Failed to import publisher "${company.name}"`,
);
continue;
}
context?.logger.info(`Imported publisher "${company.name}"`);
context?.logger.info(`Imported publisher "${companyData.name}"`);
publishers.push(res);
}
}
server/internal/metadata/index.ts
+17 -7
View File
@@ -191,10 +191,10 @@ export class MetadataHandler {
const gameId = randomUUID();
const taskId = createGameImportTaskId(libraryId, libraryPath);
await taskHandler.create({
const key = createGameImportTaskId(libraryId, libraryPath);
return await taskHandler.create({
name: `Import game "${result.name}" (${libraryPath})`,
id: taskId,
key,
taskGroup: "import:game",
acls: ["system:import:game:read"],
async run(context) {
@@ -213,6 +213,11 @@ export class MetadataHandler {
}),
);
const companyLookupCache: {
[key: string]: Awaited<
ReturnType<typeof metadataHandler.fetchCompany>
>;
} = {};
let metadata: GameMetadata | undefined = undefined;
try {
metadata = await provider.fetchGame(
@@ -220,8 +225,13 @@ export class MetadataHandler {
id: result.id,
name: result.name,
// wrap in anonymous functions to keep references to this
publisher: (name: string) => metadataHandler.fetchCompany(name),
developer: (name: string) => metadataHandler.fetchCompany(name),
company: async (name: string) => {
if (companyLookupCache[name]) return companyLookupCache[name];
const companyData = await metadataHandler.fetchCompany(name);
companyLookupCache[name] = companyData;
return companyData;
},
createObject,
},
wrapTaskContext(context, {
@@ -281,10 +291,10 @@ export class MetadataHandler {
logger.info(`Finished game import.`);
progress(100);
context.addAction(`View Game:/admin/library/${gameId}`);
},
});
return taskId;
}
// Careful with this function, it has no typechecking
server/internal/metadata/pcgamingwiki.ts
+14 -26
View File
@@ -9,14 +9,13 @@ import type {
CompanyMetadata,
GameMetadataRating,
} from "./types";
import type { AxiosRequestConfig } from "axios";
import axios from "axios";
import * as jdenticon from "jdenticon";
import { DateTime } from "luxon";
import * as cheerio from "cheerio";
import { type } from "arktype";
import type { TaskRunContext } from "../tasks";
import { logger } from "~/server/internal/logging";
import type { NitroFetchOptions, NitroFetchRequest } from "nitropack";
interface PCGamingWikiParseRawPage {
parse: {
@@ -104,35 +103,24 @@ export class PCGamingWikiProvider implements MetadataProvider {
private async request<T>(
query: URLSearchParams,
options?: AxiosRequestConfig,
options?: NitroFetchOptions<NitroFetchRequest, "get" | "post">,
) {
const finalURL = `https://www.pcgamingwiki.com/w/api.php?${query.toString()}`;
const overlay: AxiosRequestConfig = {
url: finalURL,
baseURL: "",
};
const response = await axios.request<T>(
Object.assign({}, options, overlay),
);
if (response.status !== 200)
throw new Error(
`Error in pcgamingwiki \nStatus Code: ${response.status}\n${response.data}`,
);
const response = await $fetch<T>(finalURL, options);
return response;
}
private async cargoQuery<T>(
query: URLSearchParams,
options?: AxiosRequestConfig,
options?: NitroFetchOptions<NitroFetchRequest, "get" | "post">,
) {
const response = await this.request<PCGamingWikiCargoResult<T>>(
query,
options,
);
if (response.data.error !== undefined)
if (response.error !== undefined)
throw new Error(`Error in pcgamingwiki cargo query`);
return response;
}
@@ -150,7 +138,7 @@ export class PCGamingWikiProvider implements MetadataProvider {
pageid: pageID,
});
const res = await this.request<PCGamingWikiParseRawPage>(searchParams);
const $ = cheerio.load(res.data.parse.text["*"]);
const $ = cheerio.load(res.parse.text["*"]);
// get intro based on 'introduction' class
const introductionEle = $(".introduction").first();
// remove citations from intro
@@ -281,7 +269,7 @@ export class PCGamingWikiProvider implements MetadataProvider {
await this.cargoQuery<PCGamingWikiSearchStub>(searchParams);
const results: GameMetadataSearchResult[] = [];
for (const result of response.data.cargoquery) {
for (const result of response.cargoquery) {
const game = result.title;
const pageContent = await this.getPageContent(game.PageID);
@@ -372,7 +360,7 @@ export class PCGamingWikiProvider implements MetadataProvider {
}
async fetchGame(
{ id, name, publisher, developer, createObject }: _FetchGameMetadataParams,
{ id, name, company, createObject }: _FetchGameMetadataParams,
context?: TaskRunContext,
): Promise<GameMetadata> {
context?.logger.info("Using PCGamingWiki provider");
@@ -391,10 +379,10 @@ export class PCGamingWikiProvider implements MetadataProvider {
this.cargoQuery<PCGamingWikiGame>(searchParams),
this.getPageContent(id),
]);
if (res.data.cargoquery.length < 1)
if (res.cargoquery.length < 1)
throw new Error("Error in pcgamingwiki, no game");
const game = res.data.cargoquery[0].title;
const game = res.cargoquery[0].title;
const publishers: CompanyModel[] = [];
if (game.Publishers !== null) {
@@ -403,7 +391,7 @@ export class PCGamingWikiProvider implements MetadataProvider {
for (const pub of pubListClean) {
context?.logger.info(`Importing publisher "${pub}"...`);
const res = await publisher(pub);
const res = await company(pub);
if (res === undefined) {
context?.logger.warn(`Failed to import publisher "${pub}"`);
continue;
@@ -422,7 +410,7 @@ export class PCGamingWikiProvider implements MetadataProvider {
const devListClean = this.parseWikiStringArray(game.Developers);
for (const dev of devListClean) {
context?.logger.info(`Importing developer "${dev}"...`);
const res = await developer(dev);
const res = await company(dev);
if (res === undefined) {
context?.logger.warn(`Failed to import developer "${dev}"`);
continue;
@@ -487,8 +475,8 @@ export class PCGamingWikiProvider implements MetadataProvider {
// TODO: replace with company logo
const icon = createObject(jdenticon.toPng(query, 512));
for (let i = 0; i < res.data.cargoquery.length; i++) {
const company = res.data.cargoquery[i].title;
for (let i = 0; i < res.cargoquery.length; i++) {
const company = res.cargoquery[i].title;
const fixedCompanyName =
this.parseWikiStringArray(company.PageName)[0] ?? company.PageName;
server/internal/metadata/steam.ts
+74 -60
View File
@@ -9,8 +9,8 @@ import type {
GameMetadataRating,
} from "./types";
import type { TaskRunContext } from "../tasks";
import axios from "axios";
import * as jdenticon from "jdenticon";
import { load } from "cheerio";
/**
* Note: The Steam API is largely undocumented.
@@ -188,19 +188,15 @@ export class SteamProvider implements MetadataProvider {
}
async search(query: string): Promise<GameMetadataSearchResult[]> {
const response = await axios.get<SteamSearchStub[]>(
const response = await $fetch<SteamSearchStub[]>(
`https://steamcommunity.com/actions/SearchApps/${query}`,
);
if (
response.status !== 200 ||
!response.data ||
response.data.length === 0
) {
if (!response || response.length === 0) {
return [];
}
const result: GameMetadataSearchResult[] = response.data.map((item) => ({
const result: GameMetadataSearchResult[] = response.map((item) => ({
id: item.appid,
name: item.name,
icon: item.icon || "",
@@ -208,7 +204,7 @@ export class SteamProvider implements MetadataProvider {
year: 0,
}));
const ids = response.data.map((i) => i.appid);
const ids = response.map((i) => i.appid);
const detailsResponse = await this._fetchGameDetails(ids, {
include_basic_info: true,
@@ -235,7 +231,7 @@ export class SteamProvider implements MetadataProvider {
}
async fetchGame(
{ id, publisher, developer, createObject }: _FetchGameMetadataParams,
{ id, company, createObject }: _FetchGameMetadataParams,
context?: TaskRunContext,
): Promise<GameMetadata> {
context?.logger.info(`Starting Steam metadata fetch for game ID: ${id}`);
@@ -294,38 +290,66 @@ export class SteamProvider implements MetadataProvider {
context?.progress(70);
context?.logger.info("Processing publishers and developers...");
const storePage = await $fetch<string>(
`https://store.steampowered.com/app/${id}/`,
);
const $ = load(storePage);
const companyLinks = $("a")
.toArray()
.filter(
(v) =>
v.attribs["href"]?.startsWith(
"https://store.steampowered.com/developer/",
) ||
v.attribs["href"]?.startsWith(
"https://store.steampowered.com/publisher/",
),
)
.map((v) => v.attribs.href);
const companies: {
[key: string]: {
pub: boolean;
dev: boolean;
};
} = {};
companyLinks.forEach((v) => {
const [type, name] = v
.substring("https://store.steampowered.com/".length, v.indexOf("?"))
.split("/");
companies[name] ??= { pub: false, dev: false };
switch (type) {
case "publisher":
companies[name].pub = true;
break;
case "developer":
companies[name].dev = true;
break;
}
});
const publishers = [];
const publisherNames = currentGame.basic_info.publishers || [];
context?.logger.info(
`Found ${publisherNames.length} publisher(s) to process`,
);
for (const pub of publisherNames) {
context?.logger.info(`Processing publisher: "${pub.name}"`);
const comp = await publisher(pub.name);
if (!comp) {
context?.logger.warn(`Failed to import publisher "${pub.name}"`);
continue;
}
publishers.push(comp);
context?.logger.info(`Successfully imported publisher: "${pub.name}"`);
}
const developers = [];
const developerNames = currentGame.basic_info.developers || [];
context?.logger.info(
`Found ${developerNames.length} developer(s) to process`,
);
for (const dev of developerNames) {
context?.logger.info(`Processing developer: "${dev.name}"`);
const comp = await developer(dev.name);
if (!comp) {
context?.logger.warn(`Failed to import developer "${dev.name}"`);
continue;
for (const [companyName, types] of Object.entries(companies)) {
context?.logger.info(`Processing company: "${companyName}"`);
const comp = await company(companyName);
if (types.dev) {
developers.push(comp);
context?.logger.info(
`Successfully imported developer: "${companyName}"`,
);
}
if (types.pub) {
publishers.push(comp);
context?.logger.info(
`Successfully imported publisher: "${companyName}"`,
);
}
developers.push(comp);
context?.logger.info(`Successfully imported developer: "${dev.name}"`);
}
context?.logger.info(
@@ -425,23 +449,19 @@ export class SteamProvider implements MetadataProvider {
l: "english",
});
const response = await axios.get(
`https://store.steampowered.com/developer/${query.replaceAll(" ", "")}/?${searchParams.toString()}`,
{
maxRedirects: 0,
},
);
const url = `https://store.steampowered.com/developer/${encodeURIComponent(query)}/?${searchParams.toString()}`;
const response = await $fetch<string>(url);
if (response.status !== 200 || !response.data) {
if (!response) {
return undefined;
}
const html = response.data;
const html = response;
// Extract metadata from HTML meta tags
const metadata = this._extractMetaTagsFromHtml(html);
if (!metadata.title) {
if (!metadata.title || metadata.title == "Steam Search") {
return undefined;
}
@@ -623,14 +643,12 @@ export class SteamProvider implements MetadataProvider {
}),
});
const request = await axios.get<SteamAppDetailsPackage>(
const request = await $fetch<SteamAppDetailsPackage>(
`https://api.steampowered.com/IStoreBrowseService/GetItems/v1/?${searchParams.toString()}`,
);
if (request.status !== 200) return [];
const result = [];
const storeItems = request.data?.response?.store_items ?? [];
const storeItems = request.response?.store_items ?? [];
for (const item of storeItems) {
if (item.success !== 1) continue;
@@ -723,14 +741,14 @@ export class SteamProvider implements MetadataProvider {
language,
});
const request = await axios.get<SteamTagsPackage>(
const request = await $fetch<SteamTagsPackage>(
`https://api.steampowered.com/IStoreService/GetTagList/v1/?${searchParams.toString()}`,
);
if (request.status !== 200 || !request.data.response?.tags) return [];
if (!request.response?.tags) return [];
const tagMap = new Map<number, string>();
for (const tag of request.data.response.tags) {
for (const tag of request.response.tags) {
tagMap.set(tag.tagid, tag.name);
}
@@ -756,15 +774,11 @@ export class SteamProvider implements MetadataProvider {
l: language,
});
const request = await axios.get<SteamWebAppDetailsPackage>(
const request = await $fetch<SteamWebAppDetailsPackage>(
`https://store.steampowered.com/api/appdetails?${searchParams.toString()}`,
);
if (request.status !== 200) {
return undefined;
}
const appData = request.data[appid]?.data;
const appData = request[appid]?.data;
if (!appData) {
return undefined;
}
server/internal/metadata/types.d.ts
Vendored
+1 -2
View File
@@ -65,8 +65,7 @@ export interface _FetchGameMetadataParams {
id: string;
name: string;
publisher: (query: string) => Promise<Company | undefined>;
developer: (query: string) => Promise<Company | undefined>;
company: (query: string) => Promise<Company | undefined>;
createObject: (data: TransactionDataType) => ObjectReference;
}