update Tue 07/14/2026 11:38:16.92

2026-07-14 11:38:17 +10:00
commit b2b555ef16
38 changed files with 7348 additions and 0 deletions
@@ -0,0 +1,193 @@
+/* exhibit.js — main viewer.
+ * Pipeline: camera video -> ArUco detect (maxHamming 0) -> per-marker POS-IT pose
+ * (rotation as-is, -t[1] -t[2]) -> fuseWorld (confidence-weighted slerp) -> Three.js
+ * camera placed in world -> server-driven ghosts + building occlusion meshes.
+ */
+import * as THREE from 'three';
+import { createDetector, detectMarkers, quadArea } from './ar/detect.js';
+import { PoseEstimator } from './ar/pose.js';
+import { WorldFuser } from './ar/fuse.js';
+import { buildGhost } from './ghosts/loader.js';
+import { ghostTransform } from './ghosts/behavior.js';
+import { ExhibitNet, installErrorReporter } from './net.js';
+
+const $ = (s) => document.querySelector(s);
+let info = { mode: 'dev' };
+let scene3, camera3, renderer, video, canvas2d, ctx2d;
+let detector, poseEst, fuser;
+let gradients = {}, modelManifest = null;
+const activeGhosts = new Map(); // uid -> { rec, group }
+let tracking = { markers: 0, lastSeen: 0 };
+const clockOffsetSamples = [];
+let clockOffset = 0; // serverNow - clientNow
+
+async function boot() {
+  info = await (await fetch('/api/info')).json();
+  installErrorReporter(info.mode === 'dev');
+
+  const g = await (await fetch('/api/ghosts')).json();
+  gradients = g.gradients.gradients || g.gradients;
+  modelManifest = await (await fetch('/api/models')).json();
+
+  $('#start').addEventListener('click', start, { once: true });
+}
+
+async function start() {
+  $('#startScreen').classList.add('hidden');
+  $('#hud').classList.remove('hidden');
+
+  // camera
+  video = $('#cam');
+  const stream = await navigator.mediaDevices.getUserMedia({
+    video: { facingMode: 'environment', width: { ideal: 1280 }, height: { ideal: 720 } },
+    audio: false,
+  });
+  video.srcObject = stream;
+  await video.play();
+
+  // detection canvas
+  canvas2d = document.createElement('canvas');
+  ctx2d = canvas2d.getContext('2d', { willReadFrequently: true });
+
+  // three.js
+  renderer = new THREE.WebGLRenderer({ canvas: $('#gl'), alpha: true, antialias: true });
+  renderer.setPixelRatio(Math.min(devicePixelRatio, 2));
+  scene3 = new THREE.Scene();
+  camera3 = new THREE.PerspectiveCamera(60, innerWidth / innerHeight, 0.02, 50);
+  scene3.add(new THREE.AmbientLight(0xffffff, 1.2));
+  onResize(); addEventListener('resize', onResize);
+
+  detector = createDetector();
+  fuser = new WorldFuser();
+
+  // network
+  const net = new ExhibitNet();
+  net.on('scene', (m) => { fuser.setScene(m.scene); buildOcclusion(m.scene); })
+     .on('active', async (m) => {
+       for (const uid of [...activeGhosts.keys()]) removeGhost(uid);
+       for (const rec of m.ghosts) await addGhost(rec);
+     })
+     .on('spawn', (m) => addGhost(m.ghost))
+     .on('despawn', (m) => scheduleRemove(m.uid))
+     .on('pong', (m) => {
+       const rtt = performance.now() - m.t;
+       clockOffsetSamples.push(m.server + rtt / 2 - Date.now());
+       if (clockOffsetSamples.length > 5) clockOffsetSamples.shift();
+       clockOffset = clockOffsetSamples.reduce((a, b) => a + b, 0) / clockOffsetSamples.length;
+     });
+  net.connect();
+  setInterval(() => { try { net.ws.send(JSON.stringify({ type: 'ping', t: performance.now() })); } catch {} }, 5000);
+
+  requestAnimationFrame(loop);
+}
+
+function onResize() {
+  renderer.setSize(innerWidth, innerHeight);
+  camera3.aspect = innerWidth / innerHeight;
+  camera3.updateProjectionMatrix();
+}
+
+// ---------- occlusion: invisible depth-only building volumes ----------
+const occlusionGroup = new THREE.Group();
+function buildOcclusion(scene) {
+  occlusionGroup.clear();
+  const mat = new THREE.MeshBasicMaterial({ colorWrite: false }); // depth-only
+  for (const b of scene.buildings || []) {
+    const geo = new THREE.BoxGeometry(b.size[0], b.size[1], b.size[2]);
+    const mesh = new THREE.Mesh(geo, mat);
+    mesh.position.set(b.position[0], b.position[1] + b.size[1] / 2, b.position[2]);
+    mesh.rotation.y = THREE.MathUtils.degToRad(b.yawDeg || 0);
+    mesh.renderOrder = -1;
+    occlusionGroup.add(mesh);
+  }
+  if (!occlusionGroup.parent) scene3.add(occlusionGroup);
+}
+
+// ---------- ghosts ----------
+async function addGhost(rec) {
+  if (activeGhosts.has(rec.uid)) return;
+  const group = await buildGhost(rec, gradients, modelManifest);
+  scene3.add(group);
+  activeGhosts.set(rec.uid, { rec, group });
+  toast(`${rec.name} appeared`);
+}
+function removeGhost(uid) {
+  const e = activeGhosts.get(uid);
+  if (!e) return;
+  scene3.remove(e.group);
+  activeGhosts.delete(uid);
+}
+function scheduleRemove(uid) {
+  const e = activeGhosts.get(uid);
+  if (!e) return removeGhost(uid);
+  // let the client-side crossfade (driven by rec.until) finish, then remove
+  const wait = Math.max(0, e.rec.until - (Date.now() + clockOffset)) + (e.rec.crossfade || 3) * 1000;
+  setTimeout(() => removeGhost(uid), Math.min(wait, 8000));
+}
+
+// ---------- main loop ----------
+const _tmp = { position: new THREE.Vector3(), rotationY: 0, opacity: 1 };
+let frameCount = 0;
+
+function loop(t) {
+  requestAnimationFrame(loop);
+  if (video && video.readyState >= 2) {
+    // downscale for detection speed
+    const W = 640;
+    const H = Math.round(W * video.videoHeight / video.videoWidth);
+    if (canvas2d.width !== W) { canvas2d.width = W; canvas2d.height = H; }
+    ctx2d.drawImage(video, 0, 0, W, H);
+    const img = ctx2d.getImageData(0, 0, W, H);
+
+    if (!poseEst) {
+      // focal length in detection-canvas pixels from camera FOV assumption (~60° h-fov)
+      poseEst = new PoseEstimator(W / (2 * Math.tan(THREE.MathUtils.degToRad(60) / 2)));
+    }
+
+    const markers = detectMarkers(detector, img, fuser.knownIds());
+    tracking.markers = markers.length;
+    if (markers.length) {
+      tracking.lastSeen = performance.now();
+      const estimates = markers.map(m => {
+        const e = poseEst.estimate(m.id, m.corners, W / 2, H / 2, fuser.sizeFor(m.id));
+        return e && { markerId: m.id, position: e.position, quaternion: e.quaternion, area: quadArea(m.corners) };
+      }).filter(Boolean);
+      const fused = fuser.fuse(estimates);
+      if (fused) {
+        camera3.position.copy(fused.position);
+        camera3.quaternion.copy(fused.quaternion);
+      }
+    }
+  }
+
+  // ghosts: deterministic motion on synced clock
+  const now = Date.now() + clockOffset;
+  for (const { rec, group } of activeGhosts.values()) {
+    ghostTransform(rec, now, _tmp);
+    group.position.copy(_tmp.position);
+    group.rotation.y = _tmp.rotationY;
+    group.userData.setOpacity(_tmp.opacity * 0.92);
+    group.userData.tick(t / 1000);
+  }
+
+  // HUD
+  if ((frameCount++ & 15) === 0) {
+    const stale = performance.now() - tracking.lastSeen > 1500;
+    $('#trk').textContent = stale ? 'Point at a Newbury Crest' : `Tracking ${tracking.markers} crest${tracking.markers === 1 ? '' : 's'}`;
+    $('#trk').classList.toggle('warn', stale || tracking.markers < 2);
+    $('#cnt').textContent = `${activeGhosts.size} ghost${activeGhosts.size === 1 ? '' : 's'} nearby`;
+  }
+
+  renderer.render(scene3, camera3);
+}
+
+let toastTimer = null;
+function toast(msg) {
+  const el = $('#toast');
+  el.textContent = msg;
+  el.classList.add('show');
+  clearTimeout(toastTimer);
+  toastTimer = setTimeout(() => el.classList.remove('show'), 2500);
+}
+
+boot().catch(e => { console.error(e); alert('Failed to start: ' + e.message); });