newbury-exhibit/public/js/exhibit.js

/* Newbury Exhibit — the Hidden Side (multi-viewer AR)
 *
 * This is the payoff: point a phone at any Newbury Crest and the ghosts appear
 * anchored to the physical town, moving along their authored paths, hidden behind
 * buildings — and every phone in the room sees the SAME ghosts in the SAME spots,
 * each from its own viewpoint.
 *
 * How the shared world works:
 *   1. Camera detects a marker -> POS-IT gives camera pose RELATIVE TO THAT MARKER.
 *   2. scene.json says where that marker sits in WORLD coords (anchor position/yaw).
 *   3. Compose: world->camera transform. Any device seeing any known marker lands
 *      in the same world frame, so world-space ghosts line up for everyone.
 *   4. The server streams ghost world-positions at 20Hz; we place them in the world
 *      group, which is transformed into camera space. Buildings are invisible
 *      depth-only occluders so ghosts vanish behind them.
 */
import * as THREE from 'three';
import { buildLegoHeadGhost } from './lego-head-ghost.js?v=1';
import { TunedDetector, MarkerTracker } from './detect-tuned.js?v=1';

const MM = 0.001; // world mm -> three.js metres

const $ = (id) => document.getElementById(id);
const video = $('video');
const canvas = $('three');
const startScreen = $('start'), goBtn = $('go'), errEl = $('err');
const hud = $('hud'), countEl = $('count'), lockTag = $('lock'), line2 = $('line2');
const ghostNEl = $('ghostN'), realignEl = $('realign');

const grab = document.createElement('canvas');
const gctx = grab.getContext('2d', { willReadFrequently: true });

let detector = null, tracker = null, focalPx = 0, running = false;

// ---- Three.js scene -------------------------------------------------------
const renderer = new THREE.WebGLRenderer({ canvas, alpha: true, antialias: true });
renderer.setClearColor(0x000000, 0);
renderer.sortObjects = true;

const scene3d = new THREE.Scene();
scene3d.add(new THREE.AmbientLight(0xb8c4e0, 1.15));
const key = new THREE.DirectionalLight(0xffffff, 0.75);
key.position.set(0.3, 1, 0.5); scene3d.add(key);

// Camera sits at origin looking down -Z. We move the WORLD, not the camera.
let camera = new THREE.PerspectiveCamera(55, 1, 5, 8000); // mm units (large far)

// `world` holds everything in build coordinates; its matrix = world->camera.
const world = new THREE.Group();
world.matrixAutoUpdate = false;
scene3d.add(world);

// sub-groups inside world
const gGhosts = new THREE.Group(); world.add(gGhosts);
const gOccluders = new THREE.Group(); world.add(gOccluders);
const gAnchorViz = new THREE.Group(); world.add(gAnchorViz); // subtle anchor markers

// ---- Scene data from server ----------------------------------------------
let sceneMeta = null;         // { world, anchors[], blockers[] }
const anchorById = new Map(); // fiducialId -> anchor (world placement)
const ghostObjs = new Map();  // spawnId -> { group, targetPos, lastColor }

// ---- Marker + pose --------------------------------------------------------
const MARKER_SIZE_MM = 96;    // outer black square; must match printed/built markers
let posit = null;

function centredCorners(m, w, h) {
  return m.corners.map((c) => ({ x: c.x - w / 2, y: h / 2 - c.y }));
}

// Build the marker->camera matrix (POS-IT), converting js-aruco axes to three.js.
// DERIVED FROM REAL DEVICE DATA (orient-debug log): the POS-IT rotation is already
// correct as-is. The old F=diag(1,-1,-1) premultiply was rotating the whole frame
// 180 deg about X -> pitch negated + 180 deg roll ("upside down + inverted tilt").
// The ONLY conversion needed is on the TRANSLATION: three.js's camera looks down
// -Z (and screen Y is up), so we negate translation Y and Z. Rotation stays
// untouched, which keeps pitch/roll/yaw matching the real marker exactly.
function poseToMatrix(R, t) {
  return new THREE.Matrix4().set(
    R[0][0], R[0][1], R[0][2],  t[0],
    R[1][0], R[1][1], R[1][2], -t[1],   // flip translation Y (screen up)
    R[2][0], R[2][1], R[2][2], -t[2],   // flip translation Z (in front of -Z cam)
    0, 0, 0, 1
  );
}

// POS-IT returns TWO solutions for a planar marker (the real pose and a mirror
// twin). Up close they have near-equal error and the solver flip-flops between
// them -> "rotate 90deg, ghost spins 180deg". We return both; alignWorld picks
// the one whose resulting world orientation is closest to the previous frame.
function markerToCameraMatrix(m) {
  const pose = posit.pose(centredCorners(m, grab.width, grab.height));
  if (!pose) return null;
  const best = { M: poseToMatrix(pose.bestRotation, pose.bestTranslation), err: pose.bestError };
  let alt = null;
  if (pose.alternativeRotation && pose.alternativeTranslation) {
    alt = { M: poseToMatrix(pose.alternativeRotation, pose.alternativeTranslation), err: pose.alternativeError };
  }
  const t = pose.bestTranslation;
  return { best, alt, dist: Math.hypot(t[0], t[1], t[2]) };
}

// anchor world placement -> matrix mapping the MARKER's local frame (as POS-IT
// sees it: marker in its XY plane, +Z out of the printed face) into WORLD space.
//
// A crest lying flat on the table has its face pointing UP, so the marker's
// local +Z must map to world +Y. That's a -90 deg rotation about X. On top of
// that we apply the anchor's yaw (spin on the table) and its world position.
//
//   mount 'flat' (default): marker lies on the table, face up   -> rotX(-90) then yaw about Y
//   mount 'wall':           marker stands vertical, face outward -> yaw about Y only
//
// The order matters: worldFromMarker = T(pos) * Ry(yaw) * planeTilt.
function anchorToWorldMatrix(anchor) {
  const p = anchor.position || { x: 0, y: 0, z: 0 };
  const yaw = THREE.MathUtils.degToRad(anchor.rotationDeg?.y || 0);
  const mount = anchor.mount || 'flat';

  const planeTilt = new THREE.Matrix4();
  if (mount === 'wall') planeTilt.identity();          // face already points sideways
  else planeTilt.makeRotationX(-Math.PI / 2);           // flat: tip face up to +Y

  const spin = new THREE.Matrix4().makeRotationY(yaw);
  const trans = new THREE.Matrix4().setPosition(p.x, p.y, p.z);

  // trans * spin * planeTilt
  return trans.multiply(spin).multiply(planeTilt);
}

// Given a detected marker with a known anchor, compute world->camera and apply
// it to the `world` group so everything in world coords renders correctly.
//
// Planar-ambiguity defeat: POS-IT gives two candidate poses. We compute the
// resulting world->camera for each, and keep whichever is closest to LAST
// frame's transform (temporal consistency). This stops the mirror-twin flip
// that made the ghost spin the wrong way. On first lock we take `best`.
const tmpAnchorInv = new THREE.Matrix4();
let lastWorldToCamera = null; // previous frame's chosen transform (for continuity)

function worldToCameraFor(mMatrix, anchorInv) {
  return new THREE.Matrix4().multiplyMatrices(mMatrix, anchorInv);
}

// crude distance between two 4x4s: sum of squared element differences of the
// rotation part (enough to tell the real pose from its mirror twin).
function matDist(a, b) {
  const ea = a.elements, eb = b.elements;
  let s = 0;
  for (const i of [0, 1, 2, 4, 5, 6, 8, 9, 10]) { const d = ea[i] - eb[i]; s += d * d; }
  return s;
}

function alignWorld(m2c, anchor) {
  const aToW = anchorToWorldMatrix(anchor);
  tmpAnchorInv.copy(aToW).invert();

  const candBest = worldToCameraFor(m2c.best.M, tmpAnchorInv);
  let chosen = candBest;

  if (m2c.alt) {
    const candAlt = worldToCameraFor(m2c.alt.M, tmpAnchorInv);
    if (lastWorldToCamera) {
      // pick the candidate closest to where the world was last frame
      chosen = matDist(candAlt, lastWorldToCamera) < matDist(candBest, lastWorldToCamera)
        ? candAlt : candBest;
    } else {
      // first lock: trust the lower-error (best) solution
      chosen = candBest;
    }
  }

  world.matrix.copy(chosen);
  world.matrixWorldNeedsUpdate = true;
  lastWorldToCamera = chosen.clone();
}

// ---- Multi-marker fusion --------------------------------------------------
// When several crests are visible at once, each yields its own world->camera
// estimate. Picking a single "winner" makes the world snap as detection flickers
// between them. Because every crest's world position is known and the rig is
// rigid, we instead FUSE all visible estimates into one consensus pose:
//   * for each marker, resolve its two POS-IT solutions to the one consistent
//     with last frame (defeats the planar flip), producing one world->camera;
//   * weight each by confidence (bigger, lower-error, non-coasting = more trust);
//   * average translations linearly and rotations via quaternion slerp.
// Result: smoother and more accurate than any single marker, with no snap.
const _p = new THREE.Vector3(), _q = new THREE.Quaternion(), _s = new THREE.Vector3();

function resolveMarkerWorldToCamera(m2c, anchor) {
  // returns the single world->camera for this marker, flip-resolved vs last frame
  const aToW = anchorToWorldMatrix(anchor);
  const inv = new THREE.Matrix4().copy(aToW).invert();
  const candBest = worldToCameraFor(m2c.best.M, inv);
  if (!m2c.alt) return candBest;
  const candAlt = worldToCameraFor(m2c.alt.M, inv);
  if (!lastWorldToCamera) {
    return (m2c.alt.err < m2c.best.err) ? candBest : candBest; // first lock: best
  }
  return matDist(candAlt, lastWorldToCamera) < matDist(candBest, lastWorldToCamera)
    ? candAlt : candBest;
}

function fuseWorld(observations) {
  // observations: [{ m2c, anchor, weight }]
  if (observations.length === 0) return false;

  // Resolve each to a single world->camera, decompose to pos+quat.
  const parts = [];
  for (const o of observations) {
    const wc = resolveMarkerWorldToCamera(o.m2c, o.anchor);
    const p = new THREE.Vector3(), q = new THREE.Quaternion(), s = new THREE.Vector3();
    wc.decompose(p, q, s);
    parts.push({ p, q, w: o.weight });
  }

  // Weighted translation average.
  const totalW = parts.reduce((a, b) => a + b.w, 0) || 1;
  const pos = new THREE.Vector3();
  for (const pt of parts) pos.addScaledVector(pt.p, pt.w / totalW);

  // Weighted rotation average via incremental slerp. Keep all quats on the same
  // hemisphere as the first (q and -q are the same rotation; slerp needs care).
  const q0 = parts[0].q.clone();
  let quat = q0.clone();
  let accum = parts[0].w;
  for (let i = 1; i < parts.length; i++) {
    let qi = parts[i].q.clone();
    if (q0.dot(qi) < 0) qi.set(-qi.x, -qi.y, -qi.z, -qi.w); // hemisphere align
    accum += parts[i].w;
    quat.slerp(qi, parts[i].w / accum); // incremental weighted mean
    quat.normalize();
  }

  const fused = new THREE.Matrix4().compose(pos, quat, new THREE.Vector3(1, 1, 1));
  world.matrix.copy(fused);
  world.matrixWorldNeedsUpdate = true;
  lastWorldToCamera = fused.clone();
  return true;
}

// ---- Build occluders + anchor viz once scene arrives ----------------------
function buildStaticWorld() {
  // occluders: invisible depth-only boxes so ghosts hide behind buildings
  gOccluders.clear();
  for (const bl of (sceneMeta.blockers || [])) {
    const sz = bl.size;
    const geo = new THREE.BoxGeometry(sz.x, sz.y, sz.z);
    const mat = new THREE.MeshBasicMaterial({ colorWrite: false }); // writes depth only
    const mesh = new THREE.Mesh(geo, mat);
    mesh.position.set(bl.position.x + sz.x / 2, bl.position.y + sz.y / 2, bl.position.z + sz.z / 2);
    mesh.renderOrder = -1; // draw occluders before ghosts so depth is ready
    gOccluders.add(mesh);
  }

  // very subtle anchor dots so you can see the crests are recognised (optional)
  gAnchorViz.clear();
  for (const a of (sceneMeta.anchors || [])) {
    const dot = new THREE.Mesh(
      new THREE.SphereGeometry(6, 10, 10),
      new THREE.MeshBasicMaterial({ color: 0x00e5c0, transparent: true, opacity: 0.25 })
    );
    dot.position.set(a.position.x, a.position.y, a.position.z);
    gAnchorViz.add(dot);
  }
}

// ---- Ghosts ---------------------------------------------------------------
const LURE_KEY = { Red: 'Red', Yellow: 'Yellow', Blue: 'Blue' };

function ensureGhost(g) {
  let obj = ghostObjs.get(g.spawnId);
  if (!obj) {
    const colorKey = colorForGhost(g);
    const group = buildLegoHeadGhost(THREE, colorKey);
    group.scale.setScalar(1.0); // ghost built in mm already
    gGhosts.add(group);
    obj = { group, targetPos: new THREE.Vector3(), lastColor: colorKey };
    ghostObjs.set(g.spawnId, obj);
  }
  return obj;
}

// The server sends ghostId; we map to a lure colour via the roster if we have it.
let roster = null;
function colorForGhost(g) {
  if (g.color) return LURE_KEY[g.color] || 'Blue';
  if (roster && g.ghostId && roster.has(g.ghostId)) return roster.get(g.ghostId).color || 'Blue';
  return 'Blue';
}

function updateGhosts(list, dtServer) {
  const seen = new Set();
  for (const g of list) {
    seen.add(g.spawnId);
    const obj = ensureGhost(g);
    // server pos is world mm; set as smoothing target
    obj.targetPos.set(g.pos.x, g.pos.y, g.pos.z);
    obj._yaw = THREE.MathUtils.degToRad(g.yawDeg || 0);
  }
  // remove ghosts no longer present
  for (const [id, obj] of ghostObjs) {
    if (!seen.has(id)) { gGhosts.remove(obj.group); ghostObjs.delete(id); }
  }
  ghostNEl.textContent = ghostObjs.size;
}

// ---- WebSocket sync -------------------------------------------------------
let ws = null, wsConnected = false, lastState = 0;

function connectWS() {
  const proto = location.protocol === 'https:' ? 'wss' : 'ws';
  ws = new WebSocket(`${proto}://${location.host}/sync`);
  ws.onopen = () => { wsConnected = true; setLine(); };
  ws.onclose = () => { wsConnected = false; setLine(); setTimeout(connectWS, 1500); };
  ws.onerror = () => { /* onclose will handle reconnect */ };
  ws.onmessage = (ev) => {
    let msg; try { msg = JSON.parse(ev.data); } catch (_) { return; }
    if (msg.type === 'hello') {
      sceneMeta = { world: msg.world, anchors: msg.anchors || [], blockers: msg.blockers || [] };
      anchorById.clear();
      for (const a of sceneMeta.anchors) anchorById.set(a.fiducialId, a);
      buildStaticWorld();
      setLine();
    } else if (msg.type === 'state') {
      lastState = performance.now();
      updateGhosts(msg.ghosts, msg.serverTime);
    }
  };
}

// fetch roster once for colour mapping (optional, best-effort)
async function loadRoster() {
  try {
    const r = await fetch('/api/ghosts');
    const list = await r.json();
    roster = new Map(list.map((g) => [g.id, g]));
  } catch (_) { roster = null; }
}

// ---- Alignment state ------------------------------------------------------
let aligned = false;
let lastAlign = 0;
const REALIGN_GRACE_MS = 1500; // keep last alignment briefly after marker lost

function setLock(state) {
  // state: 'locked' | 'holding' | 'searching'
  if (state === 'locked') { lockTag.textContent = 'locked'; lockTag.className = 'tag on'; realignEl.classList.remove('show'); }
  else if (state === 'holding') { lockTag.textContent = 'holding'; lockTag.className = 'tag warn'; }
  else { lockTag.textContent = 'aligning…'; lockTag.className = 'tag off'; }
}
function setLine() {
  if (!wsConnected) { line2.textContent = 'connecting to exhibit…'; return; }
  if (!sceneMeta) { line2.textContent = 'loading scene…'; return; }
  if (!aligned) { line2.textContent = 'point at a Newbury Crest'; return; }
  line2.innerHTML = `aligned · <b>${ghostObjs.size}</b> ghosts live`;
}

// ---- Camera + sizing ------------------------------------------------------
function sizeAll() {
  const vw = video.videoWidth, vh = video.videoHeight;
  if (!vw || !vh) return;
  grab.width = vw; grab.height = vh;
  const scale = Math.max(window.innerWidth / vw, window.innerHeight / vh);
  const dw = vw * scale, dh = vh * scale;
  for (const el of [video, canvas]) { el.style.width = dw + 'px'; el.style.height = dh + 'px'; }
  renderer.setPixelRatio(Math.min(window.devicePixelRatio, 2));
  renderer.setSize(dw, dh, false);
  focalPx = vw;
  const fovY = 2 * Math.atan((vh / 2) / focalPx) * (180 / Math.PI);
  camera.fov = fovY; camera.aspect = dw / dh; camera.updateProjectionMatrix();
  posit = new POS.Posit(MARKER_SIZE_MM, focalPx);
}

async function start() {
  errEl.textContent = '';
  try {
    const stream = await navigator.mediaDevices.getUserMedia({
      video: { facingMode: { ideal: 'environment' }, width: { ideal: 1280 }, height: { ideal: 720 } }, audio: false,
    });
    video.srcObject = stream; await video.play();
  } catch (e) {
    errEl.textContent = 'Camera unavailable: ' + (e.message || e.name) + '. iOS needs HTTPS + camera permission.';
    return;
  }
  detector = new TunedDetector({ dictionaryName: 'ARUCO_4X4_1000' }).setPreset('forgiving');
  tracker = new MarkerTracker({ coastMs: 120, smooth: true }); // short coast: don't linger on stale pose while panning

  await new Promise((res) => { if (video.readyState >= 2) res(); else video.onloadeddata = () => res(); });
  sizeAll();
  window.addEventListener('resize', sizeAll);

  await loadRoster();
  connectWS();

  startScreen.classList.add('hidden');
  hud.classList.remove('hidden'); countEl.classList.remove('hidden');
  running = true;
  requestAnimationFrame(loop);
}

// ---- Main loop ------------------------------------------------------------
function loop(now) {
  if (!running) return;
  requestAnimationFrame(loop);
  if (video.readyState < 2 || !grab.width) return;

  // 1) detect + track markers
  gctx.drawImage(video, 0, 0, grab.width, grab.height);
  const img = gctx.getImageData(0, 0, grab.width, grab.height);
  let raw = [];
  try { raw = detector.detect(img, { width: grab.width, height: grab.height }); } catch (_) { raw = []; }
  const markers = tracker.update(raw, now);

  // 2) Fuse ALL visible KNOWN markers into one consensus world pose.
  //    Each contributes an estimate weighted by confidence (area, low pose
  //    error, not coasting). Fusing removes the snap you get when a single
  //    "winner" marker flickers between two visible crests, and is more
  //    accurate because the rig geometry is known and rigid.
  const observations = [];
  let anyLive = false;
  for (const m of markers) {
    if (!anchorById.has(m.id)) continue;
    const m2c = markerToCameraMatrix(m);
    if (!m2c) continue;
    // confidence: lower pose error + live (not coasting) + nearer = more trust
    const errW = 1 / (1 + (m2c.best.err ?? 0));       // lower error -> higher weight
    const coastW = m.coasting ? 0.35 : 1;             // trust live reads more
    const nearW = 1 / (1 + m2c.dist / 1000);          // nearer markers slightly favoured
    const weight = errW * coastW * nearW;
    observations.push({ m2c, anchor: anchorById.get(m.id), weight });
    if (!m.coasting) anyLive = true;
  }

  if (observations.length > 0) {
    fuseWorld(observations);
    aligned = true; lastAlign = now;
    world.visible = true;
    setLock(anyLive ? 'locked' : 'holding');
  } else if (aligned && now - lastAlign < REALIGN_GRACE_MS) {
    // keep showing world briefly using last alignment
    setLock('holding');
  } else {
    aligned = false;
    world.visible = false;
    setLock('searching');
    if (sceneMeta && wsConnected) realignEl.classList.add('show');
  }
  setLine();

  // 3) animate ghosts: smooth toward server target, apply yaw + idle wobble
  const t = now / 1000;
  for (const obj of ghostObjs.values()) {
    obj.group.position.lerp(obj.targetPos, 0.5); // smooth network updates (snappier follow)
    if (obj._yaw != null) {
      // face travel/authored yaw, eased
      const cur = obj.group.rotation.y;
      let d = obj._yaw - cur; while (d > Math.PI) d -= 2 * Math.PI; while (d < -Math.PI) d += 2 * Math.PI;
      obj.group.rotation.y = cur + d * 0.15;
    }
    obj.group.userData.update?.(t);
  }

  // 4) stale-state watchdog: if server went quiet, dim the count
  if (wsConnected && now - lastState > 3000) ghostNEl.style.opacity = '0.4';
  else ghostNEl.style.opacity = '1';

  renderer.render(scene3d, camera);
}

goBtn.addEventListener('click', start);