From fc1b13cc260cec0b36ba3cbad96e95f1085056bf Mon Sep 17 00:00:00 2001 From: jessikitty Date: Wed, 19 Aug 2026 16:06:12 +1000 Subject: [PATCH] =?UTF-8?q?v3:=20ghost=20triage=20instrumentation=20?= =?UTF-8?q?=E2=80=94=20WS=20rx=20trace,=20ghost=20record=20count,=20buildG?= =?UTF-8?q?host=20error=20reporting,=20dev=20beacon,=20pose=20self-consist?= =?UTF-8?q?ency=20(aim)=20readout?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- public/js/exhibit.js | 90 ++++++++++++++++++++++++++++++++++++++------ 1 file changed, 78 insertions(+), 12 deletions(-) diff --git a/public/js/exhibit.js b/public/js/exhibit.js index ab00a18..41976ae 100644 --- a/public/js/exhibit.js +++ b/public/js/exhibit.js @@ -53,6 +53,9 @@ let gradients = {}, modelManifest = null, characters = { defaults: {}, byId: {} let ghostHeight = 4; // cm, from scene.ghostHeightCm const activeGhosts = new Map(); // uid -> { rec, group } let tracking = { markers: 0, lastSeen: 0 }; +let netRef = null; +const rxLog = []; // last few WS message types, newest last +function rxTrace(t) { rxLog.push(t); if (rxLog.length > 6) rxLog.shift(); } const clockOffsetSamples = []; let clockOffset = 0; // serverNow - clientNow @@ -64,6 +67,48 @@ const HFOV_DEG = parseFloat(localStorage.getItem('nbx.hfovDeg')) || 60; const _dbgEuler = new THREE.Euler(); const _dbgV = new THREE.Vector3(); let dbgEl = null; +let dbgPose = ''; +function dbgStatus() { + const ws = netRef && netRef.ws ? ['conn','open','closing','closed'][netRef.ws.readyState] : 'none'; + return `ws ${ws} rx ${rxLog.join(',') || '-'}\n` + + `ghost recs ${activeGhosts.size}\n`; +} +function paintDbg() { if (dbgEl) dbgEl.textContent = dbgStatus() + dbgPose + dbgAim; } + +/* Aim check: the pose is only self-consistent if the marker we can SEE in the + * camera frame sits where the solved pose projects it. Compares the marker's + * measured pixel centre against its projected position, and reports the ghost's + * on-screen NDC. |ndc| < 1 means the ghost is inside the frustum. + * px = where the crest actually is in frame (-1..1, 0 = centre) + * proj= where the pose says it should be + * Those two disagreeing is a wrong frame convention, not tracking noise. */ +let dbgAim = ''; +const _aimV = new THREE.Vector3(); +function updateAim(markers, W, H) { + if (!cvSolver) return; + let s = ''; + const m0 = markers && markers[0]; + if (m0) { + const a = cvSolver.anchors.get(m0.id); + let cxp = 0, cyp = 0; + for (const c of m0.corners) { cxp += c.x; cyp += c.y; } + cxp = (cxp / 4) / W * 2 - 1; + cyp = -((cyp / 4) / H * 2 - 1); + s += `m${m0.id} px(${cxp.toFixed(2)},${cyp.toFixed(2)})`; + if (a) { + _aimV.setFromMatrixPosition(a.mat).project(camera3); + s += ` proj(${_aimV.x.toFixed(2)},${_aimV.y.toFixed(2)})`; + } + s += '\n'; + } + const first = activeGhosts.values().next().value; + if (first) { + _aimV.copy(first.group.position).project(camera3); + const onScreen = Math.abs(_aimV.x) < 1 && Math.abs(_aimV.y) < 1 && _aimV.z < 1; + s += `ghost ndc(${_aimV.x.toFixed(2)},${_aimV.y.toFixed(2)}) ${onScreen ? 'ON-SCREEN' : 'off-screen'}`; + } + dbgAim = s; +} function updateDbg(fusedQuat, markerCount, extra) { if (!dbgEl) return; const deg = r => (r * 180 / Math.PI).toFixed(0).padStart(4); @@ -75,16 +120,17 @@ function updateDbg(fusedQuat, markerCount, extra) { const modeLine = engine === 'opencv' ? `axis ${cvSolver ? cvSolver.getAxisMode() : 'std'} cv:${cvState}${cvNote ? ' ' + cvNote : ''}` : `rot ${getRotMode()} cam ${camMode}`; - dbgEl.textContent = + dbgPose = `eng ${engine} ${freezeGhosts ? 'FROZEN' : 'moving'}\n` + `${modeLine}\n` + `markers ${markerCount}${extra}\n${rawLine}\nview ${v3(view)}\nup ${v3(up)}\n` + `pos ${v3(camera3.position)}`; + paintDbg(); } async function boot() { info = await (await fetch('/api/info')).json(); - installErrorReporter(info.mode === 'dev'); + reportError = installErrorReporter(info.mode === 'dev'); await applyLanding(); @@ -218,24 +264,26 @@ async function start() { // network const net = new ExhibitNet(); - net.on('scene', (m) => { + netRef = net; + const on = (t, fn) => net.on(t, (m) => { rxTrace(t); return fn(m); }); + on('scene', (m) => { fuser.setScene(m.scene); cvSolver.setScene(m.scene); buildOcclusion(m.scene); ghostHeight = m.scene.ghostHeightCm || 4; for (const e of activeGhosts.values()) e.group.userData.setHeight(ghostHeight); - }) - .on('characters', async (m) => { + }); + on('characters', async (m) => { characters = m.characters || characters; const recs = [...activeGhosts.values()].map(e => e.rec); for (const uid of [...activeGhosts.keys()]) removeGhost(uid); for (const rec of recs) await addGhost(rec); // rebuild with new visuals - }) - .on('active', async (m) => { + }); + on('active', async (m) => { for (const uid of [...activeGhosts.keys()]) removeGhost(uid); for (const rec of m.ghosts) await addGhost(rec); - }) - .on('spawn', (m) => addGhost(m.ghost)) - .on('despawn', (m) => scheduleRemove(m.uid)) - .on('pong', (m) => { + }); + on('spawn', (m) => addGhost(m.ghost)); + on('despawn', (m) => scheduleRemove(m.uid)); + on('pong', (m) => { const rtt = performance.now() - m.t; clockOffsetSamples.push(m.server + rtt / 2 - Date.now()); if (clockOffsetSamples.length > 5) clockOffsetSamples.shift(); @@ -272,8 +320,23 @@ function buildOcclusion(scene) { // ---------- ghosts ---------- async function addGhost(rec) { if (activeGhosts.has(rec.uid)) return; - const group = await buildGhost(rec, gradients, modelManifest, characters); + let group; + try { + group = await buildGhost(rec, gradients, modelManifest, characters); + } catch (e) { + reportError({ kind: 'ghost', msg: `buildGhost ${rec.name}: ${(e && e.message) || e}` }); + return; + } group.userData.setHeight(ghostHeight); + if (info.mode === 'dev') { + // triage beacon: always-on-top magenta dot at the ghost's origin — if the + // record exists client-side, this shows regardless of model/shader/opacity + const dot = new THREE.Mesh( + new THREE.SphereGeometry(1, 12, 8), + new THREE.MeshBasicMaterial({ color: 0xff33cc, depthTest: false, transparent: true, opacity: 0.9 })); + dot.renderOrder = 999; + group.add(dot); + } scene3.add(group); activeGhosts.set(rec.uid, { rec, group }); toast(`${rec.name} appeared`); @@ -311,6 +374,7 @@ function loop(t) { const markers = detectMarkers(detector, img, fuser.knownIds()); tracking.markers = markers.length; + if (dbgEl) updateAim(markers, W, H); if (markers.length) { tracking.lastSeen = performance.now(); @@ -355,6 +419,7 @@ function loop(t) { // HUD if ((frameCount++ & 15) === 0) { + paintDbg(); const stale = performance.now() - tracking.lastSeen > 1500; $('#trk').textContent = stale ? 'Point at a Newbury Crest' : `Tracking ${tracking.markers} crest${tracking.markers === 1 ? '' : 's'}`; $('#trk').classList.toggle('warn', stale || tracking.markers < 2); @@ -404,6 +469,7 @@ async function capturePhoto() { } } +let reportError = () => {}; let toastTimer = null; function toast(msg) { const el = $('#toast');