From 81f216871722e23aaac91dcd5886e8a6bbfe8841 Mon Sep 17 00:00:00 2001 From: jessikitty Date: Thu, 13 Aug 2026 22:42:37 +1000 Subject: [PATCH] =?UTF-8?q?v3:=20exhibit.js=20=E2=80=94=20dual=20tracking?= =?UTF-8?q?=20engines=20(opencv=20board=20solve=20default,=20posit=20A/B?= =?UTF-8?q?=20fallback)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- public/js/exhibit.js | 414 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 414 insertions(+) create mode 100644 public/js/exhibit.js diff --git a/public/js/exhibit.js b/public/js/exhibit.js new file mode 100644 index 0000000..60b8fb4 --- /dev/null +++ b/public/js/exhibit.js @@ -0,0 +1,414 @@ +/* exhibit.js — main viewer (v3). + * Pipeline: camera video -> ArUco detect (maxHamming 0) -> TRACKING ENGINE -> Three.js + * camera placed in world -> server-driven ghosts + building occlusion meshes. + * + * v3 ships two runtime-switchable tracking engines (HUD overlay, tap top third): + * opencv (default) — joint multi-marker board solve (solve-cv.js). All detected + * corners -> one solvePnP -> camera pose. Non-coplanar wall + * crest removes planar ambiguity outright. + * posit — the v2 per-marker POS-IT + fuseWorld path, kept intact for + * on-table A/B until the new engine is confirmed. Delete later. + */ +import * as THREE from 'three'; +import { createDetector, detectMarkers, quadArea } from './ar/detect.js'; +import { PoseEstimator, ROT_MODES, setRotMode, getRotMode } from './ar/pose.js'; +import { WorldFuser } from './ar/fuse.js'; +import { CvBoardSolver, CV_AXIS_MODES, loadOpenCV, isCvReady } from './ar/solve-cv.js'; +import { buildGhost } from './ghosts/loader.js'; +import { ghostTransform } from './ghosts/behavior.js'; +import { ExhibitNet, installErrorReporter } from './net.js'; +import { renderMarkdown } from './md.js'; + +const $ = (s) => document.querySelector(s); + +/* ---------------- tracking engines ---------------- */ +const ENGINES = ['opencv', 'posit']; +let engine = localStorage.getItem('nbx.engine') || 'opencv'; +if (!ENGINES.includes(engine)) engine = 'opencv'; +let cvState = 'idle'; // idle | loading | ready | failed +function setEngine(e) { engine = e; localStorage.setItem('nbx.engine', e); } + +/* Camera-frame correction for the POSIT path (unchanged from v2; the opencv path + * needs none — its conversion was validated synthetically and has its own + * axis-mode toggle in solve-cv.js for marker-frame insurance). */ +const CAM_CORR = { + none: new THREE.Quaternion(), + y180: new THREE.Quaternion().setFromAxisAngle(new THREE.Vector3(0, 1, 0), Math.PI), + x180: new THREE.Quaternion().setFromAxisAngle(new THREE.Vector3(1, 0, 0), Math.PI), + z180: new THREE.Quaternion().setFromAxisAngle(new THREE.Vector3(0, 0, 1), Math.PI), +}; +const CAM_MODES = ['none', 'y180', 'x180', 'z180']; +let camMode = 'none'; + +/* Diagnostic: freeze all ghost motion (bob/sway/wander/path) so ghosts sit at their + * static spawn position — tells tracking jitter apart from the float animation. + * Defaults ON in dev; tap the tracking HUD chip to toggle. */ +let freezeGhosts = false; + +let info = { mode: 'dev' }; +let scene3, camera3, renderer, video, canvas2d, ctx2d; +let detector, poseEst, fuser, cvSolver; +let gradients = {}, modelManifest = null, characters = { defaults: {}, byId: {} }; +let ghostHeight = 4; // cm, from scene.ghostHeightCm +const activeGhosts = new Map(); // uid -> { rec, group } +let tracking = { markers: 0, lastSeen: 0 }; +const clockOffsetSamples = []; +let clockOffset = 0; // serverNow - clientNow + +/* Horizontal FOV assumption for the focal estimate; override for a specific device + * via localStorage 'nbx.hfovDeg' (shared by both engines). */ +const HFOV_DEG = parseFloat(localStorage.getItem('nbx.hfovDeg')) || 60; + +// Debug pose readout (dev mode) +const _dbgEuler = new THREE.Euler(); +const _dbgV = new THREE.Vector3(); +let dbgEl = null; +function updateDbg(fusedQuat, markerCount, extra) { + if (!dbgEl) return; + const deg = r => (r * 180 / Math.PI).toFixed(0).padStart(4); + _dbgEuler.setFromQuaternion(fusedQuat, 'YXZ'); + const rawLine = `raw P${deg(_dbgEuler.x)} Y${deg(_dbgEuler.y)} R${deg(_dbgEuler.z)}`; + const view = _dbgV.set(0, 0, -1).applyQuaternion(camera3.quaternion).clone(); + const up = _dbgV.set(0, 1, 0).applyQuaternion(camera3.quaternion).clone(); + const v3 = v => `(${v.x.toFixed(2)},${v.y.toFixed(2)},${v.z.toFixed(2)})`; + const modeLine = engine === 'opencv' + ? `axis ${cvSolver ? cvSolver.getAxisMode() : 'std'} cv:${cvState}` + : `rot ${getRotMode()} cam ${camMode}`; + dbgEl.textContent = + `eng ${engine} ${freezeGhosts ? 'FROZEN' : 'moving'}\n` + + `${modeLine}\n` + + `markers ${markerCount}${extra}\n${rawLine}\nview ${v3(view)}\nup ${v3(up)}\n` + + `pos ${v3(camera3.position)}`; +} + +async function boot() { + info = await (await fetch('/api/info')).json(); + installErrorReporter(info.mode === 'dev'); + + await applyLanding(); + + const g = await (await fetch('/api/ghosts')).json(); + gradients = g.gradients.gradients || g.gradients; + modelManifest = await (await fetch('/api/models')).json(); + characters = await (await fetch('/api/characters')).json(); + + $('#start').addEventListener('click', start, { once: true }); + $('#detailsBtn').addEventListener('click', () => $('#details').classList.remove('hidden')); + $('#backBtn').addEventListener('click', () => $('#details').classList.add('hidden')); +} + +/* Paint the operator-authored landing page + details page. */ +async function applyLanding() { + let L; + try { L = await (await fetch('/api/landing')).json(); } catch { return; } + const set = (sel, txt) => { const el = $(sel); if (el) el.textContent = txt || ''; }; + + document.documentElement.style.setProperty('--accent', L.accent || '#51eaf1'); + document.documentElement.style.setProperty('--text', L.textColor || '#e8ecff'); + document.documentElement.style.setProperty('--ov', L.overlay != null ? L.overlay : 0.55); + + if (L.backgroundUrl) $('#startScreen').style.backgroundImage = `url("${L.backgroundUrl}")`; + if (L.logoUrl) { + const img = $('#logo'); + img.src = L.logoUrl; img.alt = L.title || 'logo'; + img.classList.remove('hidden'); + $('#title').classList.add('hidden'); // logo art replaces the text title + } else { + set('#title', L.title); + } + set('#subtitle', L.subtitle); + set('#start', L.startButton || 'Start'); + set('#detailsBtn', L.detailsButton || 'About'); + set('#detailsTitle', L.detailsTitle); + $('#detailsBody').innerHTML = renderMarkdown(L.detailsMarkdown); + set('#disc1', L.disclaimer); + set('#disc2', L.disclaimer); + if (!L.detailsMarkdown) $('#detailsBtn').classList.add('hidden'); + document.title = L.title || 'Newbury Nights'; +} + +/* Lazy-load the OpenCV.js WASM (~13MB, cached hard by the server). On failure the + * posit engine takes over automatically — visitors always get a working exhibit. */ +function ensureOpenCV() { + if (cvState === 'ready' || cvState === 'loading') return; + cvState = 'loading'; + loadOpenCV().then(() => { + cvState = 'ready'; + if (engine === 'opencv') toast('Tracking engine ready'); + }).catch((e) => { + cvState = 'failed'; + console.warn('opencv.js unavailable, falling back to posit', e); + if (engine === 'opencv') { setEngine('posit'); toast('Using classic tracking'); } + }); +} + +async function start() { + $('#startScreen').classList.add('hidden'); + $('#hud').classList.remove('hidden'); + $('#shutter').classList.remove('hidden'); + $('#shutter').addEventListener('click', capturePhoto); + + // camera + video = $('#cam'); + const stream = await navigator.mediaDevices.getUserMedia({ + video: { facingMode: 'environment', width: { ideal: 1280 }, height: { ideal: 720 } }, + audio: false, + }); + video.srcObject = stream; + await video.play(); + + // detection canvas + canvas2d = document.createElement('canvas'); + ctx2d = canvas2d.getContext('2d', { willReadFrequently: true }); + + // three.js + renderer = new THREE.WebGLRenderer({ canvas: $('#gl'), alpha: true, antialias: true, preserveDrawingBuffer: true }); + renderer.setPixelRatio(Math.min(devicePixelRatio, 2)); + scene3 = new THREE.Scene(); + camera3 = new THREE.PerspectiveCamera(60, innerWidth / innerHeight, 2, 5000); + scene3.add(new THREE.AmbientLight(0xffffff, 1.2)); + onResize(); addEventListener('resize', onResize); + + detector = createDetector(); + fuser = new WorldFuser(); + cvSolver = new CvBoardSolver(); + ensureOpenCV(); + + // dev-mode pose readout + tap-to-cycle modes + if (info.mode === 'dev') { + freezeGhosts = true; // start static so tracking jitter is isolated from ghost float + dbgEl = $('#dbg'); + if (dbgEl) { + dbgEl.style.display = 'block'; + dbgEl.style.pointerEvents = 'auto'; + /* tap zones (thirds): + * top — cycle engine (opencv <-> posit) + * middle — cycle engine mode (opencv: axis std/ymirror; posit: rot mode) + * bottom — cycle posit camera correction (no-op for opencv) */ + dbgEl.addEventListener('click', (ev) => { + const r = dbgEl.getBoundingClientRect(); + const frac = (ev.clientY - r.top) / r.height; + if (frac < 0.34) { + const next = ENGINES[(ENGINES.indexOf(engine) + 1) % ENGINES.length]; + setEngine(next); + if (next === 'opencv') ensureOpenCV(); + } else if (frac < 0.67) { + if (engine === 'opencv') { + const i = CV_AXIS_MODES.indexOf(cvSolver.getAxisMode()); + cvSolver.setAxisMode(CV_AXIS_MODES[(i + 1) % CV_AXIS_MODES.length]); + } else { + const i = ROT_MODES.indexOf(getRotMode()); + setRotMode(ROT_MODES[(i + 1) % ROT_MODES.length]); + } + } else if (engine === 'posit') { + const i = CAM_MODES.indexOf(camMode); + camMode = CAM_MODES[(i + 1) % CAM_MODES.length]; + } + }); + } + // tap the tracking HUD chip to toggle ghost motion freeze + const trk = $('#trk'); + if (trk) { + trk.style.pointerEvents = 'auto'; + trk.addEventListener('click', () => { freezeGhosts = !freezeGhosts; }); + } + } + + // network + const net = new ExhibitNet(); + net.on('scene', (m) => { + fuser.setScene(m.scene); cvSolver.setScene(m.scene); buildOcclusion(m.scene); + ghostHeight = m.scene.ghostHeightCm || 4; + for (const e of activeGhosts.values()) e.group.userData.setHeight(ghostHeight); + }) + .on('characters', async (m) => { + characters = m.characters || characters; + const recs = [...activeGhosts.values()].map(e => e.rec); + for (const uid of [...activeGhosts.keys()]) removeGhost(uid); + for (const rec of recs) await addGhost(rec); // rebuild with new visuals + }) + .on('active', async (m) => { + for (const uid of [...activeGhosts.keys()]) removeGhost(uid); + for (const rec of m.ghosts) await addGhost(rec); + }) + .on('spawn', (m) => addGhost(m.ghost)) + .on('despawn', (m) => scheduleRemove(m.uid)) + .on('pong', (m) => { + const rtt = performance.now() - m.t; + clockOffsetSamples.push(m.server + rtt / 2 - Date.now()); + if (clockOffsetSamples.length > 5) clockOffsetSamples.shift(); + clockOffset = clockOffsetSamples.reduce((a, b) => a + b, 0) / clockOffsetSamples.length; + }); + net.connect(); + setInterval(() => { try { net.ws.send(JSON.stringify({ type: 'ping', t: performance.now() })); } catch {} }, 5000); + + requestAnimationFrame(loop); +} + +function onResize() { + renderer.setSize(innerWidth, innerHeight); + camera3.aspect = innerWidth / innerHeight; + camera3.updateProjectionMatrix(); +} + +// ---------- occlusion: invisible depth-only building volumes ---------- +const occlusionGroup = new THREE.Group(); +function buildOcclusion(scene) { + occlusionGroup.clear(); + const mat = new THREE.MeshBasicMaterial({ colorWrite: false }); // depth-only + for (const b of scene.buildings || []) { + const geo = new THREE.BoxGeometry(b.size[0], b.size[1], b.size[2]); + const mesh = new THREE.Mesh(geo, mat); + mesh.position.set(b.position[0], b.position[1] + b.size[1] / 2, b.position[2]); + mesh.rotation.y = THREE.MathUtils.degToRad(b.yawDeg || 0); + mesh.renderOrder = -1; + occlusionGroup.add(mesh); + } + if (!occlusionGroup.parent) scene3.add(occlusionGroup); +} + +// ---------- ghosts ---------- +async function addGhost(rec) { + if (activeGhosts.has(rec.uid)) return; + const group = await buildGhost(rec, gradients, modelManifest, characters); + group.userData.setHeight(ghostHeight); + scene3.add(group); + activeGhosts.set(rec.uid, { rec, group }); + toast(`${rec.name} appeared`); +} +function removeGhost(uid) { + const e = activeGhosts.get(uid); + if (!e) return; + scene3.remove(e.group); + activeGhosts.delete(uid); +} +function scheduleRemove(uid) { + const e = activeGhosts.get(uid); + if (!e) return removeGhost(uid); + const until = e.rec.until ?? (Date.now() + clockOffset); + const wait = Math.max(0, until - (Date.now() + clockOffset)) + (e.rec.crossfade || 3) * 1000; + setTimeout(() => removeGhost(uid), Math.min(wait, 8000)); +} + +// ---------- main loop ---------- +const _tmp = { position: new THREE.Vector3(), rotationY: 0, opacity: 1 }; +let frameCount = 0; + +function loop(t) { + requestAnimationFrame(loop); + if (video && video.readyState >= 2) { + // downscale for detection speed; corner precision scales with resolution. + const W = 960; + const H = Math.round(W * video.videoHeight / video.videoWidth); + if (canvas2d.width !== W) { canvas2d.width = W; canvas2d.height = H; } + ctx2d.drawImage(video, 0, 0, W, H); + const img = ctx2d.getImageData(0, 0, W, H); + + const focal = W / (2 * Math.tan(THREE.MathUtils.degToRad(HFOV_DEG) / 2)); + if (!poseEst) poseEst = new PoseEstimator(focal); + + const markers = detectMarkers(detector, img, fuser.knownIds()); + tracking.markers = markers.length; + if (markers.length) { + tracking.lastSeen = performance.now(); + + if (engine === 'opencv' && cvState === 'ready') { + // v3 path: one joint solve over every visible crest + const fused = cvSolver.solve(markers, W, H, focal); + if (fused) { + camera3.position.copy(fused.position); + camera3.quaternion.copy(fused.quaternion); + updateDbg(fused.quaternion, fused.markerCount, ` reproj ${fused.reprojPx.toFixed(1)}px`); + } + } else { + // v2 path: per-marker POS-IT -> confidence-weighted fusion + const estimates = markers.map(m => { + const e = poseEst.estimate(m.id, m.corners, W / 2, H / 2, fuser.sizeFor(m.id)); + return e && { markerId: m.id, position: e.position, quaternion: e.quaternion, area: quadArea(m.corners) }; + }).filter(Boolean); + const fused = fuser.fuse(estimates); + if (fused) { + camera3.position.copy(fused.position); + camera3.quaternion.copy(fused.quaternion).multiply(CAM_CORR[camMode]); + updateDbg(fused.quaternion, fused.markerCount, fused.markerCount > 1 ? ` spread ${fused.spread.toFixed(1)}cm` : ''); + } + } + } + } + + // ghosts: deterministic motion on synced clock + const now = Date.now() + clockOffset; + for (const { rec, group } of activeGhosts.values()) { + ghostTransform(rec, now, _tmp); + if (freezeGhosts) { + group.position.set(rec.pos[0], rec.pos[1], rec.pos[2]); + group.rotation.y = 0; + } else { + group.position.copy(_tmp.position); + group.rotation.y = _tmp.rotationY; + } + group.userData.setOpacity(_tmp.opacity * 0.92); + group.userData.tick(freezeGhosts ? 0 : t / 1000); + } + + // HUD + if ((frameCount++ & 15) === 0) { + const stale = performance.now() - tracking.lastSeen > 1500; + $('#trk').textContent = stale ? 'Point at a Newbury Crest' : `Tracking ${tracking.markers} crest${tracking.markers === 1 ? '' : 's'}`; + $('#trk').classList.toggle('warn', stale || tracking.markers < 2); + $('#cnt').textContent = `${activeGhosts.size} ghost${activeGhosts.size === 1 ? '' : 's'} nearby`; + } + + renderer.render(scene3, camera3); +} + +/* Shutter: composite the live camera frame with the rendered ghosts and save a PNG. */ +async function capturePhoto() { + try { + const flash = $('#flash'); + flash.classList.add('on'); + setTimeout(() => flash.classList.remove('on'), 60); + + const vw = video.videoWidth, vh = video.videoHeight; + const out = document.createElement('canvas'); + out.width = vw; out.height = vh; + const c = out.getContext('2d'); + c.drawImage(video, 0, 0, vw, vh); + + const gl = $('#gl'); + const scale = Math.max(vw / gl.width, vh / gl.height); + const dw = gl.width * scale, dh = gl.height * scale; + renderer.render(scene3, camera3); // ensure the buffer is current + c.drawImage(gl, (vw - dw) / 2, (vh - dh) / 2, dw, dh); + + const stamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19); + const blob = await new Promise(r => out.toBlob(r, 'image/png')); + const file = new File([blob], `newbury-${stamp}.png`, { type: 'image/png' }); + + if (navigator.canShare && navigator.canShare({ files: [file] })) { + await navigator.share({ files: [file], title: 'Newbury Nights' }); + toast('Photo ready to share'); + } else { + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; a.download = file.name; a.click(); + setTimeout(() => URL.revokeObjectURL(url), 10000); + toast('Photo saved'); + } + } catch (e) { + if (e && e.name === 'AbortError') return; // user dismissed the share sheet + toast('Could not save photo'); + console.warn('capture failed', e); + } +} + +let toastTimer = null; +function toast(msg) { + const el = $('#toast'); + el.textContent = msg; + el.classList.add('show'); + clearTimeout(toastTimer); + toastTimer = setTimeout(() => el.classList.remove('show'), 2500); +} + +boot().catch(e => { console.error(e); alert('Failed to start: ' + e.message); });