jessikitty/newbury-exhibit-v3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: FOV applies to the image's long axis (iOS gives portrait streams — focal was 1.78x short); cap detection canvas long side (portrait was 1.6MP not 0.5MP); show duplicate detections in HUD

Browse Source
This commit is contained in:
jessikitty
2026-08-24 11:26:45 +10:00
parent f2ee880bf1
commit 2c78c48bcc
1 changed files with 28 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
public/js/exhibit.js
+28 -14
View File
@@ -59,7 +59,7 @@ let detector, poseEst, fuser, cvSolver;
let gradients = {}, modelManifest = null, characters = { defaults: {}, byId: {} };
let ghostHeight = 4; // cm, from scene.ghostHeightCm
const activeGhosts = new Map(); // uid -> { rec, group }
let tracking = { markers: 0, raw: 0, rawIds: [], lastSeen: 0 };
let tracking = { markers: 0, raw: 0, rawIds: [], dupes: 0, lastSeen: 0 };
let netRef = null;
let sceneLoaded = false;
let anchorCount = 0;
@@ -68,9 +68,17 @@ function rxTrace(t) { rxLog.push(t); if (rxLog.length > 6) rxLog.shift(); }
const clockOffsetSamples = [];
let clockOffset = 0; // serverNow - clientNow
/* Horizontal FOV assumption for the focal estimate; override for a specific device
* via localStorage 'nbx.hfovDeg' (shared by both engines). */
/* Field-of-view assumption behind every focal estimate; override per device via
* localStorage 'nbx.hfovDeg' (shared by both engines).
*
* IT APPLIES TO THE LONG AXIS OF THE IMAGE, NOT TO videoWidth. iOS can hand back
* a PORTRAIT stream (720x1280), and a phone lens has its wide field across the
* sensor's long side either way. Assuming the angle spanned videoWidth made the
* focal length 1280/720 = 1.78x too short on portrait streams — the exact factor
* measured on device, and the reason ghosts still slid around after the camera
* model was "fixed". */
const HFOV_DEG = parseFloat(localStorage.getItem('nbx.hfovDeg')) || 60;
const focalPxFor = (w, h) => (Math.max(w, h) / 2) / Math.tan(THREE.MathUtils.degToRad(HFOV_DEG) / 2);
/* Detection cadence. Marker detection (getImageData + ArUco decode over a 960px
* frame) is by far the most expensive thing per frame, and running it on every
@@ -147,7 +155,7 @@ function updateDbg(fusedQuat, markerCount, extra) {
dbgPose =
`eng ${engine} ${freezeGhosts ? 'FROZEN' : 'moving'}\n` +
`${modeLine}\n` +
`markers ${markerCount}/${tracking.raw} seen [${tracking.rawIds.join(',')}]${extra}\n${rawLine}\nview ${v3(view)}\nup ${v3(up)}\n` +
`markers ${markerCount}/${tracking.raw} seen [${tracking.rawIds.join(',')}]${tracking.dupes ? ' +' + tracking.dupes + 'dup' : ''}${extra}\n${rawLine}\nview ${v3(view)}\nup ${v3(up)}\n` +
`pos ${v3(camera3.position)}`;
paintDbg();
}
@@ -346,10 +354,11 @@ function onResize() {
* geometry project too far from centre — measured at 1.7x on device, which
* reads as ghosts sliding the wrong way when you tilt.
*
* So: take the focal length implied by HFOV_DEG over the full video width, work
* out how much of the frame survives the cover-crop, and set the fov from that
* visible height. Cover crops symmetrically, so the principal point stays
* centred and no lens shift is needed. */
* So: take the focal length implied by HFOV_DEG over the image's LONG axis (see
* focalPxFor — the stream may be portrait), work out how much of the frame
* survives the cover-crop, and set the fov from that visible height. Cover crops
* symmetrically, so the principal point stays centred and no lens shift is
* needed. */
const _crop = { x: 1, y: 1 }; // fraction of the video width/height still visible
function updateCameraIntrinsics() {
if (!camera3) return;
@@ -357,7 +366,7 @@ function updateCameraIntrinsics() {
if (!video || !video.videoWidth) { camera3.updateProjectionMatrix(); return; }
const vw = video.videoWidth, vh = video.videoHeight;
const focal = (vw / 2) / Math.tan(THREE.MathUtils.degToRad(HFOV_DEG) / 2);
const focal = focalPxFor(vw, vh);
const scale = Math.max(innerWidth / vw, innerHeight / vh); // object-fit: cover
const visW = Math.min(vw, innerWidth / scale);
const visH = Math.min(vh, innerHeight / scale);
@@ -463,20 +472,25 @@ function loop(t) {
if (video && video.readyState >= 2 && (t - lastDetectAt) >= DETECT_INTERVAL_MS) {
lastDetectAt = t;
// downscale for detection speed; corner precision scales with resolution.
const W = 960;
const H = Math.round(W * video.videoHeight / video.videoWidth);
if (canvas2d.width !== W) { canvas2d.width = W; canvas2d.height = H; }
/* Downscale for detection speed, capping the LONG side. Fixing the WIDTH at
* 960 quietly tripled the work on a portrait stream (960x1707 = 1.6M px vs
* 960x540 = 0.5M), which is most of the cost of a frame. */
const vw = video.videoWidth, vh = video.videoHeight;
const LONG = 960;
const W = vw >= vh ? LONG : Math.round(LONG * vw / vh);
const H = vw >= vh ? Math.round(LONG * vh / vw) : LONG;
if (canvas2d.width !== W || canvas2d.height !== H) { canvas2d.width = W; canvas2d.height = H; }
ctx2d.drawImage(video, 0, 0, W, H);
const img = ctx2d.getImageData(0, 0, W, H);
const focal = W / (2 * Math.tan(THREE.MathUtils.degToRad(HFOV_DEG) / 2));
const focal = focalPxFor(W, H);
if (!poseEst) poseEst = new PoseEstimator(focal);
const markers = detectMarkers(detector, img, fuser.knownIds());
tracking.markers = markers.length;
tracking.raw = markers.rawCount || 0;
tracking.rawIds = markers.rawIds || [];
tracking.dupes = markers.dupes || 0;
if (dbgEl) updateAim(markers, W, H);
if (markers.length) {
tracking.lastSeen = performance.now();