/* Newbury Exhibit — 3D LEGO-head ghost AR driver * * Camera frame -> ArUco detection -> POS-IT 6-DoF pose -> Three.js transform. * The ghost is placed in a Three.js scene whose camera sits at the origin; we * apply the marker's solved rotation+translation to the ghost so it stays fixed * in world space. Walk around the marker and you see the head's sides — it turns. * * This is the real-3D successor to ar-test.js (which was a flat sprite). */ import * as THREE from 'three'; import { buildLegoHeadGhost } from './lego-head-ghost.js?v=1'; import { TunedDetector, MarkerTracker } from './detect-tuned.js?v=1'; const MARKER_SIZE_MM = 96; // outer black square; match what you build/print. const $ = (id) => document.getElementById(id); const video = $('video'); const canvas = $('three'); const startScreen = $('start'), goBtn = $('go'), errEl = $('err'); const hud = $('hud'), picker = $('picker'); const lockTag = $('lock'), midEl = $('mid'), distEl = $('dist'), fpsEl = $('fps'); const grab = document.createElement('canvas'); const gctx = grab.getContext('2d', { willReadFrequently: true }); let detector = null, tracker = null, posit = null, focalPx = 0, running = false; // ---- Three.js setup ------------------------------------------------------- const renderer = new THREE.WebGLRenderer({ canvas, alpha: true, antialias: true }); renderer.setClearColor(0x000000, 0); // transparent over video const scene = new THREE.Scene(); scene.add(new THREE.AmbientLight(0xb8c4e0, 1.1)); const key = new THREE.DirectionalLight(0xffffff, 0.8); key.position.set(0.3, 1, 0.6); scene.add(key); // Camera at origin looking down -Z (we move the ghost, not the camera). let camera = new THREE.PerspectiveCamera(50, 1, 1, 5000); // units: mm camera.position.set(0, 0, 0); // The ghost head, parented to an "anchor" object we drive from pose. const anchor = new THREE.Object3D(); scene.add(anchor); let ghost = buildLegoHeadGhost(THREE, 'Blue'); ghost.userData.setBaseY(MARKER_SIZE_MM * 0.9); // float above the plaque anchor.add(ghost); anchor.visible = false; // tether line marker->ghost so it reads as bound to the crest const tetherMat = new THREE.LineBasicMaterial({ color: 0x51eaf1, transparent: true, opacity: 0.35 }); const tetherGeo = new THREE.BufferGeometry().setFromPoints([ new THREE.Vector3(0, 0, 0), new THREE.Vector3(0, MARKER_SIZE_MM * 0.65, 0), ]); const tether = new THREE.Line(tetherGeo, tetherMat); anchor.add(tether); // ---- Pose -> matrix ------------------------------------------------------- // js-aruco POS-IT gives rotation (3x3) + translation (mm) of the marker in a // camera frame with X right, Y up, Z toward viewer. Three.js camera looks down // -Z with Y up, so we flip Y and Z to convert. Marker plane is the anchor's XZ. const poseMatrix = new THREE.Matrix4(); const flip = new THREE.Matrix4().makeScale(1, -1, -1); function centredCorners(m, w, h) { return m.corners.map((c) => ({ x: c.x - w / 2, y: h / 2 - c.y })); } function applyPose(m) { const pose = posit.pose(centredCorners(m, grab.width, grab.height)); if (!pose) return false; const R = pose.bestRotation, t = pose.bestTranslation; // Build matrix: rows of R, translation in last column. poseMatrix.set( R[0][0], R[0][1], R[0][2], t[0], R[1][0], R[1][1], R[1][2], t[1], R[2][0], R[2][1], R[2][2], t[2], 0, 0, 0, 1 ); // Convert axis convention. poseMatrix.premultiply(flip); anchor.matrixAutoUpdate = false; anchor.matrix.copy(poseMatrix); // The marker's own plane is XY in POS-IT; we want the ghost to stand UP off // the table, i.e. along the marker normal. Rotate the head so its +Y aligns // with the marker normal (marker's local +Z). ghost.rotation.x = -Math.PI / 2; // stand head up out of the marker plane tether.rotation.x = -Math.PI / 2; return { dist: Math.hypot(t[0], t[1], t[2]) }; } // ---- sizing --------------------------------------------------------------- function sizeAll() { const vw = video.videoWidth, vh = video.videoHeight; if (!vw || !vh) return; grab.width = vw; grab.height = vh; const scale = Math.max(window.innerWidth / vw, window.innerHeight / vh); const dw = vw * scale, dh = vh * scale; for (const el of [video, canvas]) { el.style.width = dw + 'px'; el.style.height = dh + 'px'; } renderer.setPixelRatio(Math.min(window.devicePixelRatio, 2)); renderer.setSize(dw, dh, false); camera.aspect = dw / dh; // Approximate focal length; vertical FOV from focal & sensor height in px. focalPx = vw; const fovY = 2 * Math.atan((vh / 2) / focalPx) * (180 / Math.PI); camera.fov = fovY * (dh / vh) / (dw / vw < 1 ? 1 : 1); // keep simple camera.fov = fovY; camera.updateProjectionMatrix(); posit = new POS.Posit(MARKER_SIZE_MM, focalPx); } // ---- start ---------------------------------------------------------------- async function start() { errEl.textContent = ''; try { const stream = await navigator.mediaDevices.getUserMedia({ video: { facingMode: { ideal: 'environment' }, width: { ideal: 1280 }, height: { ideal: 720 } }, audio: false, }); video.srcObject = stream; await video.play(); } catch (e) { errEl.textContent = 'Camera unavailable: ' + (e.message || e.name) + '. iOS needs HTTPS + camera permission.'; return; } detector = new TunedDetector({ dictionaryName: 'ARUCO_4X4_1000' }).setPreset('forgiving'); tracker = new MarkerTracker({ coastMs: 280, smooth: true }); await new Promise((res) => { if (video.readyState >= 2) res(); else video.onloadeddata = () => res(); }); sizeAll(); window.addEventListener('resize', sizeAll); startScreen.classList.add('hidden'); hud.classList.remove('hidden'); picker.classList.remove('hidden'); running = true; requestAnimationFrame(loop); } // ---- loop ----------------------------------------------------------------- let frames = 0, fpsStamp = performance.now(); function loop(now) { if (!running) return; requestAnimationFrame(loop); if (video.readyState < 2 || !grab.width) return; gctx.drawImage(video, 0, 0, grab.width, grab.height); const img = gctx.getImageData(0, 0, grab.width, grab.height); let raw = []; try { raw = detector.detect(img, { width: grab.width, height: grab.height }); } catch (_) { raw = []; } // coast through dropouts + smooth corners const markers = tracker.update(raw, now); if (markers.length) { // nearest-ish: smallest marker id-agnostic; just take first track const m = markers[0]; const res = applyPose(m); if (res) { anchor.visible = true; if (m.coasting) { lockTag.textContent = 'holding…'; lockTag.className = 'tag on'; } else { lockTag.textContent = 'locked'; lockTag.className = 'tag on'; } midEl.textContent = m.id; distEl.textContent = Math.round(res.dist); } } else { anchor.visible = false; lockTag.textContent = 'searching…'; lockTag.className = 'tag off'; midEl.textContent = '—'; distEl.textContent = '—'; } ghost.userData.update(now / 1000); renderer.render(scene, camera); frames++; if (now - fpsStamp > 500) { fpsEl.textContent = Math.round((frames * 1000) / (now - fpsStamp)); frames = 0; fpsStamp = now; } } // ---- colour picker -------------------------------------------------------- picker.addEventListener('click', (e) => { const sw = e.target.closest('.swatch'); if (!sw) return; picker.querySelectorAll('.swatch').forEach((s) => s.classList.remove('sel')); sw.classList.add('sel'); ghost.userData.setColor(sw.dataset.c); }); goBtn.addEventListener('click', start); // live sensitivity switch const presetSel = document.getElementById('preset'); if (presetSel) presetSel.addEventListener('change', () => { if (detector) detector.setPreset(presetSel.value); });