newbury-exhibit-v2/public/js/exhibit.js

/* exhibit.js — main viewer.
 * Pipeline: camera video -> ArUco detect (maxHamming 0) -> per-marker POS-IT pose
 * (rotation as-is, -t[1] -t[2]) -> fuseWorld (confidence-weighted slerp) -> Three.js
 * camera placed in world -> server-driven ghosts + building occlusion meshes.
 */
import * as THREE from 'three';
import { createDetector, detectMarkers, quadArea } from './ar/detect.js';
import { PoseEstimator } from './ar/pose.js';
import { WorldFuser } from './ar/fuse.js';
import { buildGhost } from './ghosts/loader.js';
import { ghostTransform } from './ghosts/behavior.js';
import { ExhibitNet, installErrorReporter } from './net.js';

const $ = (s) => document.querySelector(s);
let info = { mode: 'dev' };
let scene3, camera3, renderer, video, canvas2d, ctx2d;
let detector, poseEst, fuser;
let gradients = {}, modelManifest = null;
const activeGhosts = new Map(); // uid -> { rec, group }
let tracking = { markers: 0, lastSeen: 0 };
const clockOffsetSamples = [];
let clockOffset = 0; // serverNow - clientNow

async function boot() {
  info = await (await fetch('/api/info')).json();
  installErrorReporter(info.mode === 'dev');

  const g = await (await fetch('/api/ghosts')).json();
  gradients = g.gradients.gradients || g.gradients;
  modelManifest = await (await fetch('/api/models')).json();

  $('#start').addEventListener('click', start, { once: true });
}

async function start() {
  $('#startScreen').classList.add('hidden');
  $('#hud').classList.remove('hidden');

  // camera
  video = $('#cam');
  const stream = await navigator.mediaDevices.getUserMedia({
    video: { facingMode: 'environment', width: { ideal: 1280 }, height: { ideal: 720 } },
    audio: false,
  });
  video.srcObject = stream;
  await video.play();

  // detection canvas
  canvas2d = document.createElement('canvas');
  ctx2d = canvas2d.getContext('2d', { willReadFrequently: true });

  // three.js
  renderer = new THREE.WebGLRenderer({ canvas: $('#gl'), alpha: true, antialias: true });
  renderer.setPixelRatio(Math.min(devicePixelRatio, 2));
  scene3 = new THREE.Scene();
  camera3 = new THREE.PerspectiveCamera(60, innerWidth / innerHeight, 2, 5000);
  scene3.add(new THREE.AmbientLight(0xffffff, 1.2));
  onResize(); addEventListener('resize', onResize);

  detector = createDetector();
  fuser = new WorldFuser();

  // network
  const net = new ExhibitNet();
  net.on('scene', (m) => { fuser.setScene(m.scene); buildOcclusion(m.scene); })
     .on('active', async (m) => {
       for (const uid of [...activeGhosts.keys()]) removeGhost(uid);
       for (const rec of m.ghosts) await addGhost(rec);
     })
     .on('spawn', (m) => addGhost(m.ghost))
     .on('despawn', (m) => scheduleRemove(m.uid))
     .on('pong', (m) => {
       const rtt = performance.now() - m.t;
       clockOffsetSamples.push(m.server + rtt / 2 - Date.now());
       if (clockOffsetSamples.length > 5) clockOffsetSamples.shift();
       clockOffset = clockOffsetSamples.reduce((a, b) => a + b, 0) / clockOffsetSamples.length;
     });
  net.connect();
  setInterval(() => { try { net.ws.send(JSON.stringify({ type: 'ping', t: performance.now() })); } catch {} }, 5000);

  requestAnimationFrame(loop);
}

function onResize() {
  renderer.setSize(innerWidth, innerHeight);
  camera3.aspect = innerWidth / innerHeight;
  camera3.updateProjectionMatrix();
}

// ---------- occlusion: invisible depth-only building volumes ----------
const occlusionGroup = new THREE.Group();
function buildOcclusion(scene) {
  occlusionGroup.clear();
  const mat = new THREE.MeshBasicMaterial({ colorWrite: false }); // depth-only
  for (const b of scene.buildings || []) {
    const geo = new THREE.BoxGeometry(b.size[0], b.size[1], b.size[2]);
    const mesh = new THREE.Mesh(geo, mat);
    mesh.position.set(b.position[0], b.position[1] + b.size[1] / 2, b.position[2]);
    mesh.rotation.y = THREE.MathUtils.degToRad(b.yawDeg || 0);
    mesh.renderOrder = -1;
    occlusionGroup.add(mesh);
  }
  if (!occlusionGroup.parent) scene3.add(occlusionGroup);
}

// ---------- ghosts ----------
async function addGhost(rec) {
  if (activeGhosts.has(rec.uid)) return;
  const group = await buildGhost(rec, gradients, modelManifest);
  scene3.add(group);
  activeGhosts.set(rec.uid, { rec, group });
  toast(`${rec.name} appeared`);
}
function removeGhost(uid) {
  const e = activeGhosts.get(uid);
  if (!e) return;
  scene3.remove(e.group);
  activeGhosts.delete(uid);
}
function scheduleRemove(uid) {
  const e = activeGhosts.get(uid);
  if (!e) return removeGhost(uid);
  // let the client-side crossfade (driven by rec.until) finish, then remove
  const until = e.rec.until ?? (Date.now() + clockOffset); // residents removed by config change: fade now
  const wait = Math.max(0, until - (Date.now() + clockOffset)) + (e.rec.crossfade || 3) * 1000;
  setTimeout(() => removeGhost(uid), Math.min(wait, 8000));
}

// ---------- main loop ----------
const _tmp = { position: new THREE.Vector3(), rotationY: 0, opacity: 1 };
let frameCount = 0;

function loop(t) {
  requestAnimationFrame(loop);
  if (video && video.readyState >= 2) {
    // downscale for detection speed
    const W = 640;
    const H = Math.round(W * video.videoHeight / video.videoWidth);
    if (canvas2d.width !== W) { canvas2d.width = W; canvas2d.height = H; }
    ctx2d.drawImage(video, 0, 0, W, H);
    const img = ctx2d.getImageData(0, 0, W, H);

    if (!poseEst) {
      // focal length in detection-canvas pixels from camera FOV assumption (~60° h-fov)
      poseEst = new PoseEstimator(W / (2 * Math.tan(THREE.MathUtils.degToRad(60) / 2)));
    }

    const markers = detectMarkers(detector, img, fuser.knownIds());
    tracking.markers = markers.length;
    if (markers.length) {
      tracking.lastSeen = performance.now();
      const estimates = markers.map(m => {
        const e = poseEst.estimate(m.id, m.corners, W / 2, H / 2, fuser.sizeFor(m.id));
        return e && { markerId: m.id, position: e.position, quaternion: e.quaternion, area: quadArea(m.corners) };
      }).filter(Boolean);
      const fused = fuser.fuse(estimates);
      if (fused) {
        camera3.position.copy(fused.position);
        camera3.quaternion.copy(fused.quaternion);
      }
    }
  }

  // ghosts: deterministic motion on synced clock
  const now = Date.now() + clockOffset;
  for (const { rec, group } of activeGhosts.values()) {
    ghostTransform(rec, now, _tmp);
    group.position.copy(_tmp.position);
    group.rotation.y = _tmp.rotationY;
    group.userData.setOpacity(_tmp.opacity * 0.92);
    group.userData.tick(t / 1000);
  }

  // HUD
  if ((frameCount++ & 15) === 0) {
    const stale = performance.now() - tracking.lastSeen > 1500;
    $('#trk').textContent = stale ? 'Point at a Newbury Crest' : `Tracking ${tracking.markers} crest${tracking.markers === 1 ? '' : 's'}`;
    $('#trk').classList.toggle('warn', stale || tracking.markers < 2);
    $('#cnt').textContent = `${activeGhosts.size} ghost${activeGhosts.size === 1 ? '' : 's'} nearby`;
  }

  renderer.render(scene3, camera3);
}

let toastTimer = null;
function toast(msg) {
  const el = $('#toast');
  el.textContent = msg;
  el.classList.add('show');
  clearTimeout(toastTimer);
  toastTimer = setTimeout(() => el.classList.remove('show'), 2500);
}

boot().catch(e => { console.error(e); alert('Failed to start: ' + e.message); });