/* exhibit.js — main viewer. * Pipeline: camera video -> ArUco detect (maxHamming 0) -> per-marker POS-IT pose * (rotation as-is, -t[1] -t[2]) -> fuseWorld (confidence-weighted slerp) -> Three.js * camera placed in world -> server-driven ghosts + building occlusion meshes. */ import * as THREE from 'three'; import { createDetector, detectMarkers, quadArea } from './ar/detect.js'; import { PoseEstimator } from './ar/pose.js'; import { WorldFuser } from './ar/fuse.js'; import { buildGhost } from './ghosts/loader.js'; import { ghostTransform } from './ghosts/behavior.js'; import { ExhibitNet, installErrorReporter } from './net.js'; const $ = (s) => document.querySelector(s); let info = { mode: 'dev' }; let scene3, camera3, renderer, video, canvas2d, ctx2d; let detector, poseEst, fuser; let gradients = {}, modelManifest = null; let ghostHeight = 4; // cm, from scene.ghostHeightCm const activeGhosts = new Map(); // uid -> { rec, group } let tracking = { markers: 0, lastSeen: 0 }; const clockOffsetSamples = []; let clockOffset = 0; // serverNow - clientNow async function boot() { info = await (await fetch('/api/info')).json(); installErrorReporter(info.mode === 'dev'); const g = await (await fetch('/api/ghosts')).json(); gradients = g.gradients.gradients || g.gradients; modelManifest = await (await fetch('/api/models')).json(); $('#start').addEventListener('click', start, { once: true }); } async function start() { $('#startScreen').classList.add('hidden'); $('#hud').classList.remove('hidden'); // camera video = $('#cam'); const stream = await navigator.mediaDevices.getUserMedia({ video: { facingMode: 'environment', width: { ideal: 1280 }, height: { ideal: 720 } }, audio: false, }); video.srcObject = stream; await video.play(); // detection canvas canvas2d = document.createElement('canvas'); ctx2d = canvas2d.getContext('2d', { willReadFrequently: true }); // three.js renderer = new THREE.WebGLRenderer({ canvas: $('#gl'), alpha: true, antialias: true }); renderer.setPixelRatio(Math.min(devicePixelRatio, 2)); scene3 = new THREE.Scene(); camera3 = new THREE.PerspectiveCamera(60, innerWidth / innerHeight, 2, 5000); scene3.add(new THREE.AmbientLight(0xffffff, 1.2)); onResize(); addEventListener('resize', onResize); detector = createDetector(); fuser = new WorldFuser(); // network const net = new ExhibitNet(); net.on('scene', (m) => { fuser.setScene(m.scene); buildOcclusion(m.scene); ghostHeight = m.scene.ghostHeightCm || 4; for (const e of activeGhosts.values()) e.group.userData.setHeight(ghostHeight); }) .on('active', async (m) => { for (const uid of [...activeGhosts.keys()]) removeGhost(uid); for (const rec of m.ghosts) await addGhost(rec); }) .on('spawn', (m) => addGhost(m.ghost)) .on('despawn', (m) => scheduleRemove(m.uid)) .on('pong', (m) => { const rtt = performance.now() - m.t; clockOffsetSamples.push(m.server + rtt / 2 - Date.now()); if (clockOffsetSamples.length > 5) clockOffsetSamples.shift(); clockOffset = clockOffsetSamples.reduce((a, b) => a + b, 0) / clockOffsetSamples.length; }); net.connect(); setInterval(() => { try { net.ws.send(JSON.stringify({ type: 'ping', t: performance.now() })); } catch {} }, 5000); requestAnimationFrame(loop); } function onResize() { renderer.setSize(innerWidth, innerHeight); camera3.aspect = innerWidth / innerHeight; camera3.updateProjectionMatrix(); } // ---------- occlusion: invisible depth-only building volumes ---------- const occlusionGroup = new THREE.Group(); function buildOcclusion(scene) { occlusionGroup.clear(); const mat = new THREE.MeshBasicMaterial({ colorWrite: false }); // depth-only for (const b of scene.buildings || []) { const geo = new THREE.BoxGeometry(b.size[0], b.size[1], b.size[2]); const mesh = new THREE.Mesh(geo, mat); mesh.position.set(b.position[0], b.position[1] + b.size[1] / 2, b.position[2]); mesh.rotation.y = THREE.MathUtils.degToRad(b.yawDeg || 0); mesh.renderOrder = -1; occlusionGroup.add(mesh); } if (!occlusionGroup.parent) scene3.add(occlusionGroup); } // ---------- ghosts ---------- async function addGhost(rec) { if (activeGhosts.has(rec.uid)) return; const group = await buildGhost(rec, gradients, modelManifest); group.userData.setHeight(ghostHeight); scene3.add(group); activeGhosts.set(rec.uid, { rec, group }); toast(`${rec.name} appeared`); } function removeGhost(uid) { const e = activeGhosts.get(uid); if (!e) return; scene3.remove(e.group); activeGhosts.delete(uid); } function scheduleRemove(uid) { const e = activeGhosts.get(uid); if (!e) return removeGhost(uid); // let the client-side crossfade (driven by rec.until) finish, then remove const until = e.rec.until ?? (Date.now() + clockOffset); // residents removed by config change: fade now const wait = Math.max(0, until - (Date.now() + clockOffset)) + (e.rec.crossfade || 3) * 1000; setTimeout(() => removeGhost(uid), Math.min(wait, 8000)); } // ---------- main loop ---------- const _tmp = { position: new THREE.Vector3(), rotationY: 0, opacity: 1 }; let frameCount = 0; function loop(t) { requestAnimationFrame(loop); if (video && video.readyState >= 2) { // downscale for detection speed const W = 640; const H = Math.round(W * video.videoHeight / video.videoWidth); if (canvas2d.width !== W) { canvas2d.width = W; canvas2d.height = H; } ctx2d.drawImage(video, 0, 0, W, H); const img = ctx2d.getImageData(0, 0, W, H); if (!poseEst) { // focal length in detection-canvas pixels from camera FOV assumption (~60° h-fov) poseEst = new PoseEstimator(W / (2 * Math.tan(THREE.MathUtils.degToRad(60) / 2))); } const markers = detectMarkers(detector, img, fuser.knownIds()); tracking.markers = markers.length; if (markers.length) { tracking.lastSeen = performance.now(); const estimates = markers.map(m => { const e = poseEst.estimate(m.id, m.corners, W / 2, H / 2, fuser.sizeFor(m.id)); return e && { markerId: m.id, position: e.position, quaternion: e.quaternion, area: quadArea(m.corners) }; }).filter(Boolean); const fused = fuser.fuse(estimates); if (fused) { camera3.position.copy(fused.position); camera3.quaternion.copy(fused.quaternion); } } } // ghosts: deterministic motion on synced clock const now = Date.now() + clockOffset; for (const { rec, group } of activeGhosts.values()) { ghostTransform(rec, now, _tmp); group.position.copy(_tmp.position); group.rotation.y = _tmp.rotationY; group.userData.setOpacity(_tmp.opacity * 0.92); group.userData.tick(t / 1000); } // HUD if ((frameCount++ & 15) === 0) { const stale = performance.now() - tracking.lastSeen > 1500; $('#trk').textContent = stale ? 'Point at a Newbury Crest' : `Tracking ${tracking.markers} crest${tracking.markers === 1 ? '' : 's'}`; $('#trk').classList.toggle('warn', stale || tracking.markers < 2); $('#cnt').textContent = `${activeGhosts.size} ghost${activeGhosts.size === 1 ? '' : 's'} nearby`; } renderer.render(scene3, camera3); } let toastTimer = null; function toast(msg) { const el = $('#toast'); el.textContent = msg; el.classList.add('show'); clearTimeout(toastTimer); toastTimer = setTimeout(() => el.classList.remove('show'), 2500); } boot().catch(e => { console.error(e); alert('Failed to start: ' + e.message); });