newbury-exhibit/public/js/ar-head.js

/* Newbury Exhibit — 3D LEGO-head ghost AR driver
 *
 * Camera frame -> ArUco detection -> POS-IT 6-DoF pose -> Three.js transform.
 * The ghost is placed in a Three.js scene whose camera sits at the origin; we
 * apply the marker's solved rotation+translation to the ghost so it stays fixed
 * in world space. Walk around the marker and you see the head's sides — it turns.
 *
 * This is the real-3D successor to ar-test.js (which was a flat sprite).
 */
import * as THREE from 'three';
import { buildLegoHeadGhost } from './lego-head-ghost.js?v=1';
import { TunedDetector, MarkerTracker } from './detect-tuned.js?v=1';

const MARKER_SIZE_MM = 96; // outer black square; match what you build/print.

const $ = (id) => document.getElementById(id);
const video = $('video');
const canvas = $('three');
const startScreen = $('start'), goBtn = $('go'), errEl = $('err');
const hud = $('hud'), picker = $('picker');
const lockTag = $('lock'), midEl = $('mid'), distEl = $('dist'), fpsEl = $('fps');

const grab = document.createElement('canvas');
const gctx = grab.getContext('2d', { willReadFrequently: true });

let detector = null, tracker = null, posit = null, focalPx = 0, running = false;

// ---- Three.js setup -------------------------------------------------------
const renderer = new THREE.WebGLRenderer({ canvas, alpha: true, antialias: true });
renderer.setClearColor(0x000000, 0); // transparent over video

const scene = new THREE.Scene();
scene.add(new THREE.AmbientLight(0xb8c4e0, 1.1));
const key = new THREE.DirectionalLight(0xffffff, 0.8);
key.position.set(0.3, 1, 0.6);
scene.add(key);

// Camera at origin looking down -Z (we move the ghost, not the camera).
let camera = new THREE.PerspectiveCamera(50, 1, 1, 5000); // units: mm
camera.position.set(0, 0, 0);

// The ghost head, parented to an "anchor" object we drive from pose.
const anchor = new THREE.Object3D();
scene.add(anchor);
let ghost = buildLegoHeadGhost(THREE, 'Blue');
ghost.userData.setBaseY(MARKER_SIZE_MM * 0.9); // float above the plaque
anchor.add(ghost);
anchor.visible = false;

// tether line marker->ghost so it reads as bound to the crest
const tetherMat = new THREE.LineBasicMaterial({ color: 0x51eaf1, transparent: true, opacity: 0.35 });
const tetherGeo = new THREE.BufferGeometry().setFromPoints([
  new THREE.Vector3(0, 0, 0), new THREE.Vector3(0, MARKER_SIZE_MM * 0.65, 0),
]);
const tether = new THREE.Line(tetherGeo, tetherMat);
anchor.add(tether);

// ---- Pose -> matrix -------------------------------------------------------
// js-aruco POS-IT gives rotation (3x3) + translation (mm) of the marker in a
// camera frame with X right, Y up, Z toward viewer. Three.js camera looks down
// -Z with Y up, so we flip Y and Z to convert. Marker plane is the anchor's XZ.
const poseMatrix = new THREE.Matrix4();
const flip = new THREE.Matrix4().makeScale(1, -1, -1);

function centredCorners(m, w, h) {
  return m.corners.map((c) => ({ x: c.x - w / 2, y: h / 2 - c.y }));
}

function applyPose(m) {
  const pose = posit.pose(centredCorners(m, grab.width, grab.height));
  if (!pose) return false;
  const R = pose.bestRotation, t = pose.bestTranslation;
  // Build matrix: rows of R, translation in last column.
  poseMatrix.set(
    R[0][0], R[0][1], R[0][2], t[0],
    R[1][0], R[1][1], R[1][2], t[1],
    R[2][0], R[2][1], R[2][2], t[2],
    0, 0, 0, 1
  );
  // Convert axis convention.
  poseMatrix.premultiply(flip);
  anchor.matrixAutoUpdate = false;
  anchor.matrix.copy(poseMatrix);
  // The marker's own plane is XY in POS-IT; we want the ghost to stand UP off
  // the table, i.e. along the marker normal. Rotate the head so its +Y aligns
  // with the marker normal (marker's local +Z).
  ghost.rotation.x = -Math.PI / 2; // stand head up out of the marker plane
  tether.rotation.x = -Math.PI / 2;
  return { dist: Math.hypot(t[0], t[1], t[2]) };
}

// ---- sizing ---------------------------------------------------------------
function sizeAll() {
  const vw = video.videoWidth, vh = video.videoHeight;
  if (!vw || !vh) return;
  grab.width = vw; grab.height = vh;
  const scale = Math.max(window.innerWidth / vw, window.innerHeight / vh);
  const dw = vw * scale, dh = vh * scale;
  for (const el of [video, canvas]) { el.style.width = dw + 'px'; el.style.height = dh + 'px'; }
  renderer.setPixelRatio(Math.min(window.devicePixelRatio, 2));
  renderer.setSize(dw, dh, false);
  camera.aspect = dw / dh;
  // Approximate focal length; vertical FOV from focal & sensor height in px.
  focalPx = vw;
  const fovY = 2 * Math.atan((vh / 2) / focalPx) * (180 / Math.PI);
  camera.fov = fovY * (dh / vh) / (dw / vw < 1 ? 1 : 1); // keep simple
  camera.fov = fovY;
  camera.updateProjectionMatrix();
  posit = new POS.Posit(MARKER_SIZE_MM, focalPx);
}

// ---- start ----------------------------------------------------------------
async function start() {
  errEl.textContent = '';
  try {
    const stream = await navigator.mediaDevices.getUserMedia({
      video: { facingMode: { ideal: 'environment' }, width: { ideal: 1280 }, height: { ideal: 720 } }, audio: false,
    });
    video.srcObject = stream; await video.play();
  } catch (e) {
    errEl.textContent = 'Camera unavailable: ' + (e.message || e.name) + '. iOS needs HTTPS + camera permission.';
    return;
  }
  detector = new TunedDetector({ dictionaryName: 'ARUCO_4X4_1000' }).setPreset('forgiving');
  tracker = new MarkerTracker({ coastMs: 280, smooth: true });
  await new Promise((res) => { if (video.readyState >= 2) res(); else video.onloadeddata = () => res(); });
  sizeAll();
  window.addEventListener('resize', sizeAll);
  startScreen.classList.add('hidden');
  hud.classList.remove('hidden'); picker.classList.remove('hidden');
  running = true; requestAnimationFrame(loop);
}

// ---- loop -----------------------------------------------------------------
let frames = 0, fpsStamp = performance.now();

function loop(now) {
  if (!running) return;
  requestAnimationFrame(loop);
  if (video.readyState < 2 || !grab.width) return;

  gctx.drawImage(video, 0, 0, grab.width, grab.height);
  const img = gctx.getImageData(0, 0, grab.width, grab.height);
  let raw = [];
  try { raw = detector.detect(img, { width: grab.width, height: grab.height }); } catch (_) { raw = []; }

  // coast through dropouts + smooth corners
  const markers = tracker.update(raw, now);

  if (markers.length) {
    // nearest-ish: smallest marker id-agnostic; just take first track
    const m = markers[0];
    const res = applyPose(m);
    if (res) {
      anchor.visible = true;
      if (m.coasting) {
        lockTag.textContent = 'holding…'; lockTag.className = 'tag on';
      } else {
        lockTag.textContent = 'locked'; lockTag.className = 'tag on';
      }
      midEl.textContent = m.id;
      distEl.textContent = Math.round(res.dist);
    }
  } else {
    anchor.visible = false;
    lockTag.textContent = 'searching…'; lockTag.className = 'tag off';
    midEl.textContent = '—'; distEl.textContent = '—';
  }

  ghost.userData.update(now / 1000);
  renderer.render(scene, camera);

  frames++;
  if (now - fpsStamp > 500) { fpsEl.textContent = Math.round((frames * 1000) / (now - fpsStamp)); frames = 0; fpsStamp = now; }
}

// ---- colour picker --------------------------------------------------------
picker.addEventListener('click', (e) => {
  const sw = e.target.closest('.swatch'); if (!sw) return;
  picker.querySelectorAll('.swatch').forEach((s) => s.classList.remove('sel'));
  sw.classList.add('sel');
  ghost.userData.setColor(sw.dataset.c);
});

goBtn.addEventListener('click', start);

// live sensitivity switch
const presetSel = document.getElementById('preset');
if (presetSel) presetSel.addEventListener('change', () => {
  if (detector) detector.setPreset(presetSel.value);
});