Add exhibit AR view engine (world-aligned, server-synced ghosts, occlusion)
This commit is contained in:
@@ -0,0 +1,332 @@
|
||||
/* Newbury Exhibit — the Hidden Side (multi-viewer AR)
|
||||
*
|
||||
* This is the payoff: point a phone at any Newbury Crest and the ghosts appear
|
||||
* anchored to the physical town, moving along their authored paths, hidden behind
|
||||
* buildings — and every phone in the room sees the SAME ghosts in the SAME spots,
|
||||
* each from its own viewpoint.
|
||||
*
|
||||
* How the shared world works:
|
||||
* 1. Camera detects a marker -> POS-IT gives camera pose RELATIVE TO THAT MARKER.
|
||||
* 2. scene.json says where that marker sits in WORLD coords (anchor position/yaw).
|
||||
* 3. Compose: world->camera transform. Any device seeing any known marker lands
|
||||
* in the same world frame, so world-space ghosts line up for everyone.
|
||||
* 4. The server streams ghost world-positions at 20Hz; we place them in the world
|
||||
* group, which is transformed into camera space. Buildings are invisible
|
||||
* depth-only occluders so ghosts vanish behind them.
|
||||
*/
|
||||
import * as THREE from 'three';
|
||||
import { buildLegoHeadGhost } from './lego-head-ghost.js?v=1';
|
||||
import { TunedDetector, MarkerTracker } from './detect-tuned.js?v=1';
|
||||
|
||||
const MM = 0.001; // world mm -> three.js metres
|
||||
|
||||
const $ = (id) => document.getElementById(id);
|
||||
const video = $('video');
|
||||
const canvas = $('three');
|
||||
const startScreen = $('start'), goBtn = $('go'), errEl = $('err');
|
||||
const hud = $('hud'), countEl = $('count'), lockTag = $('lock'), line2 = $('line2');
|
||||
const ghostNEl = $('ghostN'), realignEl = $('realign');
|
||||
|
||||
const grab = document.createElement('canvas');
|
||||
const gctx = grab.getContext('2d', { willReadFrequently: true });
|
||||
|
||||
let detector = null, tracker = null, focalPx = 0, running = false;
|
||||
|
||||
// ---- Three.js scene -------------------------------------------------------
|
||||
const renderer = new THREE.WebGLRenderer({ canvas, alpha: true, antialias: true });
|
||||
renderer.setClearColor(0x000000, 0);
|
||||
renderer.sortObjects = true;
|
||||
|
||||
const scene3d = new THREE.Scene();
|
||||
scene3d.add(new THREE.AmbientLight(0xb8c4e0, 1.15));
|
||||
const key = new THREE.DirectionalLight(0xffffff, 0.75);
|
||||
key.position.set(0.3, 1, 0.5); scene3d.add(key);
|
||||
|
||||
// Camera sits at origin looking down -Z. We move the WORLD, not the camera.
|
||||
let camera = new THREE.PerspectiveCamera(55, 1, 5, 8000); // mm units (large far)
|
||||
|
||||
// `world` holds everything in build coordinates; its matrix = world->camera.
|
||||
const world = new THREE.Group();
|
||||
world.matrixAutoUpdate = false;
|
||||
scene3d.add(world);
|
||||
|
||||
// sub-groups inside world
|
||||
const gGhosts = new THREE.Group(); world.add(gGhosts);
|
||||
const gOccluders = new THREE.Group(); world.add(gOccluders);
|
||||
const gAnchorViz = new THREE.Group(); world.add(gAnchorViz); // subtle anchor markers
|
||||
|
||||
// ---- Scene data from server ----------------------------------------------
|
||||
let sceneMeta = null; // { world, anchors[], blockers[] }
|
||||
const anchorById = new Map(); // fiducialId -> anchor (world placement)
|
||||
const ghostObjs = new Map(); // spawnId -> { group, targetPos, lastColor }
|
||||
|
||||
// ---- Marker + pose --------------------------------------------------------
|
||||
const MARKER_SIZE_MM = 96; // outer black square; must match printed/built markers
|
||||
let posit = null;
|
||||
|
||||
function centredCorners(m, w, h) {
|
||||
return m.corners.map((c) => ({ x: c.x - w / 2, y: h / 2 - c.y }));
|
||||
}
|
||||
|
||||
// Build the marker->camera matrix (POS-IT), converting js-aruco axes to three.js.
|
||||
const flipYZ = new THREE.Matrix4().makeScale(1, -1, -1);
|
||||
function markerToCameraMatrix(m) {
|
||||
const pose = posit.pose(centredCorners(m, grab.width, grab.height));
|
||||
if (!pose) return null;
|
||||
const R = pose.bestRotation, t = pose.bestTranslation;
|
||||
const M = new THREE.Matrix4().set(
|
||||
R[0][0], R[0][1], R[0][2], t[0],
|
||||
R[1][0], R[1][1], R[1][2], t[1],
|
||||
R[2][0], R[2][1], R[2][2], t[2],
|
||||
0, 0, 0, 1
|
||||
);
|
||||
M.premultiply(flipYZ);
|
||||
return { M, err: pose.bestError, dist: Math.hypot(t[0], t[1], t[2]) };
|
||||
}
|
||||
|
||||
// anchor world placement -> matrix (world coords of the marker's centre + yaw).
|
||||
// The marker lies flat-ish on the build; we treat its plane with the anchor yaw
|
||||
// about the vertical (Y) axis. Anchor.position is the marker centre in world mm.
|
||||
function anchorToWorldMatrix(anchor) {
|
||||
const p = anchor.position || { x: 0, y: 0, z: 0 };
|
||||
const yaw = THREE.MathUtils.degToRad(anchor.rotationDeg?.y || 0);
|
||||
const m = new THREE.Matrix4();
|
||||
m.makeRotationY(yaw);
|
||||
m.setPosition(p.x, p.y, p.z);
|
||||
return m;
|
||||
}
|
||||
|
||||
// Given a detected marker with a known anchor, compute world->camera and apply
|
||||
// it to the `world` group so everything in world coords renders correctly.
|
||||
const tmpAnchorInv = new THREE.Matrix4();
|
||||
function alignWorld(m2c, anchor) {
|
||||
// camera = markerToCamera * anchorToWorld^-1 (as a world->camera transform)
|
||||
// world point -> anchor-local -> camera:
|
||||
// worldToCamera = markerToCamera * (anchorToWorld)^-1
|
||||
const aToW = anchorToWorldMatrix(anchor);
|
||||
tmpAnchorInv.copy(aToW).invert();
|
||||
const worldToCamera = new THREE.Matrix4().multiplyMatrices(m2c.M, tmpAnchorInv);
|
||||
world.matrix.copy(worldToCamera);
|
||||
world.matrixWorldNeedsUpdate = true;
|
||||
}
|
||||
|
||||
// ---- Build occluders + anchor viz once scene arrives ----------------------
|
||||
function buildStaticWorld() {
|
||||
// occluders: invisible depth-only boxes so ghosts hide behind buildings
|
||||
gOccluders.clear();
|
||||
for (const bl of (sceneMeta.blockers || [])) {
|
||||
const sz = bl.size;
|
||||
const geo = new THREE.BoxGeometry(sz.x, sz.y, sz.z);
|
||||
const mat = new THREE.MeshBasicMaterial({ colorWrite: false }); // writes depth only
|
||||
const mesh = new THREE.Mesh(geo, mat);
|
||||
mesh.position.set(bl.position.x + sz.x / 2, bl.position.y + sz.y / 2, bl.position.z + sz.z / 2);
|
||||
mesh.renderOrder = -1; // draw occluders before ghosts so depth is ready
|
||||
gOccluders.add(mesh);
|
||||
}
|
||||
|
||||
// very subtle anchor dots so you can see the crests are recognised (optional)
|
||||
gAnchorViz.clear();
|
||||
for (const a of (sceneMeta.anchors || [])) {
|
||||
const dot = new THREE.Mesh(
|
||||
new THREE.SphereGeometry(6, 10, 10),
|
||||
new THREE.MeshBasicMaterial({ color: 0x00e5c0, transparent: true, opacity: 0.25 })
|
||||
);
|
||||
dot.position.set(a.position.x, a.position.y, a.position.z);
|
||||
gAnchorViz.add(dot);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Ghosts ---------------------------------------------------------------
|
||||
const LURE_KEY = { Red: 'Red', Yellow: 'Yellow', Blue: 'Blue' };
|
||||
|
||||
function ensureGhost(g) {
|
||||
let obj = ghostObjs.get(g.spawnId);
|
||||
if (!obj) {
|
||||
const colorKey = colorForGhost(g);
|
||||
const group = buildLegoHeadGhost(THREE, colorKey);
|
||||
group.scale.setScalar(1.0); // ghost built in mm already
|
||||
gGhosts.add(group);
|
||||
obj = { group, targetPos: new THREE.Vector3(), lastColor: colorKey };
|
||||
ghostObjs.set(g.spawnId, obj);
|
||||
}
|
||||
return obj;
|
||||
}
|
||||
|
||||
// The server sends ghostId; we map to a lure colour via the roster if we have it.
|
||||
let roster = null;
|
||||
function colorForGhost(g) {
|
||||
if (g.color) return LURE_KEY[g.color] || 'Blue';
|
||||
if (roster && g.ghostId && roster.has(g.ghostId)) return roster.get(g.ghostId).color || 'Blue';
|
||||
return 'Blue';
|
||||
}
|
||||
|
||||
function updateGhosts(list, dtServer) {
|
||||
const seen = new Set();
|
||||
for (const g of list) {
|
||||
seen.add(g.spawnId);
|
||||
const obj = ensureGhost(g);
|
||||
// server pos is world mm; set as smoothing target
|
||||
obj.targetPos.set(g.pos.x, g.pos.y, g.pos.z);
|
||||
obj._yaw = THREE.MathUtils.degToRad(g.yawDeg || 0);
|
||||
}
|
||||
// remove ghosts no longer present
|
||||
for (const [id, obj] of ghostObjs) {
|
||||
if (!seen.has(id)) { gGhosts.remove(obj.group); ghostObjs.delete(id); }
|
||||
}
|
||||
ghostNEl.textContent = ghostObjs.size;
|
||||
}
|
||||
|
||||
// ---- WebSocket sync -------------------------------------------------------
|
||||
let ws = null, wsConnected = false, lastState = 0;
|
||||
|
||||
function connectWS() {
|
||||
const proto = location.protocol === 'https:' ? 'wss' : 'ws';
|
||||
ws = new WebSocket(`${proto}://${location.host}/sync`);
|
||||
ws.onopen = () => { wsConnected = true; setLine(); };
|
||||
ws.onclose = () => { wsConnected = false; setLine(); setTimeout(connectWS, 1500); };
|
||||
ws.onerror = () => { /* onclose will handle reconnect */ };
|
||||
ws.onmessage = (ev) => {
|
||||
let msg; try { msg = JSON.parse(ev.data); } catch (_) { return; }
|
||||
if (msg.type === 'hello') {
|
||||
sceneMeta = { world: msg.world, anchors: msg.anchors || [], blockers: msg.blockers || [] };
|
||||
anchorById.clear();
|
||||
for (const a of sceneMeta.anchors) anchorById.set(a.fiducialId, a);
|
||||
buildStaticWorld();
|
||||
setLine();
|
||||
} else if (msg.type === 'state') {
|
||||
lastState = performance.now();
|
||||
updateGhosts(msg.ghosts, msg.serverTime);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
// fetch roster once for colour mapping (optional, best-effort)
|
||||
async function loadRoster() {
|
||||
try {
|
||||
const r = await fetch('/api/ghosts');
|
||||
const list = await r.json();
|
||||
roster = new Map(list.map((g) => [g.id, g]));
|
||||
} catch (_) { roster = null; }
|
||||
}
|
||||
|
||||
// ---- Alignment state ------------------------------------------------------
|
||||
let aligned = false;
|
||||
let lastAlign = 0;
|
||||
const REALIGN_GRACE_MS = 1500; // keep last alignment briefly after marker lost
|
||||
|
||||
function setLock(state) {
|
||||
// state: 'locked' | 'holding' | 'searching'
|
||||
if (state === 'locked') { lockTag.textContent = 'locked'; lockTag.className = 'tag on'; realignEl.classList.remove('show'); }
|
||||
else if (state === 'holding') { lockTag.textContent = 'holding'; lockTag.className = 'tag warn'; }
|
||||
else { lockTag.textContent = 'aligning…'; lockTag.className = 'tag off'; }
|
||||
}
|
||||
function setLine() {
|
||||
if (!wsConnected) { line2.textContent = 'connecting to exhibit…'; return; }
|
||||
if (!sceneMeta) { line2.textContent = 'loading scene…'; return; }
|
||||
if (!aligned) { line2.textContent = 'point at a Newbury Crest'; return; }
|
||||
line2.innerHTML = `aligned · <b>${ghostObjs.size}</b> ghosts live`;
|
||||
}
|
||||
|
||||
// ---- Camera + sizing ------------------------------------------------------
|
||||
function sizeAll() {
|
||||
const vw = video.videoWidth, vh = video.videoHeight;
|
||||
if (!vw || !vh) return;
|
||||
grab.width = vw; grab.height = vh;
|
||||
const scale = Math.max(window.innerWidth / vw, window.innerHeight / vh);
|
||||
const dw = vw * scale, dh = vh * scale;
|
||||
for (const el of [video, canvas]) { el.style.width = dw + 'px'; el.style.height = dh + 'px'; }
|
||||
renderer.setPixelRatio(Math.min(window.devicePixelRatio, 2));
|
||||
renderer.setSize(dw, dh, false);
|
||||
focalPx = vw;
|
||||
const fovY = 2 * Math.atan((vh / 2) / focalPx) * (180 / Math.PI);
|
||||
camera.fov = fovY; camera.aspect = dw / dh; camera.updateProjectionMatrix();
|
||||
posit = new POS.Posit(MARKER_SIZE_MM, focalPx);
|
||||
}
|
||||
|
||||
async function start() {
|
||||
errEl.textContent = '';
|
||||
try {
|
||||
const stream = await navigator.mediaDevices.getUserMedia({
|
||||
video: { facingMode: { ideal: 'environment' }, width: { ideal: 1280 }, height: { ideal: 720 } }, audio: false,
|
||||
});
|
||||
video.srcObject = stream; await video.play();
|
||||
} catch (e) {
|
||||
errEl.textContent = 'Camera unavailable: ' + (e.message || e.name) + '. iOS needs HTTPS + camera permission.';
|
||||
return;
|
||||
}
|
||||
detector = new TunedDetector({ dictionaryName: 'ARUCO_4X4_1000' }).setPreset('forgiving');
|
||||
tracker = new MarkerTracker({ coastMs: 300, smooth: true });
|
||||
|
||||
await new Promise((res) => { if (video.readyState >= 2) res(); else video.onloadeddata = () => res(); });
|
||||
sizeAll();
|
||||
window.addEventListener('resize', sizeAll);
|
||||
|
||||
await loadRoster();
|
||||
connectWS();
|
||||
|
||||
startScreen.classList.add('hidden');
|
||||
hud.classList.remove('hidden'); countEl.classList.remove('hidden');
|
||||
running = true;
|
||||
requestAnimationFrame(loop);
|
||||
}
|
||||
|
||||
// ---- Main loop ------------------------------------------------------------
|
||||
function loop(now) {
|
||||
if (!running) return;
|
||||
requestAnimationFrame(loop);
|
||||
if (video.readyState < 2 || !grab.width) return;
|
||||
|
||||
// 1) detect + track markers
|
||||
gctx.drawImage(video, 0, 0, grab.width, grab.height);
|
||||
const img = gctx.getImageData(0, 0, grab.width, grab.height);
|
||||
let raw = [];
|
||||
try { raw = detector.detect(img, { width: grab.width, height: grab.height }); } catch (_) { raw = []; }
|
||||
const markers = tracker.update(raw, now);
|
||||
|
||||
// 2) pick the best KNOWN marker (one whose id is an anchor) to align the world.
|
||||
// Prefer the nearest, non-coasting, recognised marker.
|
||||
let best = null;
|
||||
for (const m of markers) {
|
||||
if (!anchorById.has(m.id)) continue;
|
||||
const m2c = markerToCameraMatrix(m);
|
||||
if (!m2c) continue;
|
||||
if (!best || (!m.coasting && m2c.dist < best.dist)) best = { m, m2c, dist: m2c.dist, coasting: m.coasting };
|
||||
}
|
||||
|
||||
if (best) {
|
||||
alignWorld(best.m2c, anchorById.get(best.m.id));
|
||||
aligned = true; lastAlign = now;
|
||||
world.visible = true;
|
||||
setLock(best.coasting ? 'holding' : 'locked');
|
||||
} else if (aligned && now - lastAlign < REALIGN_GRACE_MS) {
|
||||
// keep showing world briefly using last alignment
|
||||
setLock('holding');
|
||||
} else {
|
||||
aligned = false;
|
||||
world.visible = false;
|
||||
setLock('searching');
|
||||
if (sceneMeta && wsConnected) realignEl.classList.add('show');
|
||||
}
|
||||
setLine();
|
||||
|
||||
// 3) animate ghosts: smooth toward server target, apply yaw + idle wobble
|
||||
const t = now / 1000;
|
||||
for (const obj of ghostObjs.values()) {
|
||||
obj.group.position.lerp(obj.targetPos, 0.25); // smooth network updates
|
||||
if (obj._yaw != null) {
|
||||
// face travel/authored yaw, eased
|
||||
const cur = obj.group.rotation.y;
|
||||
let d = obj._yaw - cur; while (d > Math.PI) d -= 2 * Math.PI; while (d < -Math.PI) d += 2 * Math.PI;
|
||||
obj.group.rotation.y = cur + d * 0.15;
|
||||
}
|
||||
obj.group.userData.update?.(t);
|
||||
}
|
||||
|
||||
// 4) stale-state watchdog: if server went quiet, dim the count
|
||||
if (wsConnected && now - lastState > 3000) ghostNEl.style.opacity = '0.4';
|
||||
else ghostNEl.style.opacity = '1';
|
||||
|
||||
renderer.render(scene3d, camera);
|
||||
}
|
||||
|
||||
goBtn.addEventListener('click', start);
|
||||
Reference in New Issue
Block a user