From 7fc2b02e6a8b3ddb38c26f96ef214ca2dca5d10f Mon Sep 17 00:00:00 2001 From: jessikitty Date: Mon, 12 Oct 2026 13:07:46 +1100 Subject: [PATCH] AR: sub-pixel corner refinement, ROI-first detection, anchor coplanarity check --- .claude-context/00-overview.md | 1 + .claude-context/01-architecture.md | 70 ++++- .../2026-10-07-corner-precision-roi.md | 154 ++++++++++ README.md | 14 + docs/PAGES.md | 28 ++ public/js/ar/detect.js | 23 +- public/js/ar/geom.js | 107 +++++++ public/js/ar/roi.js | 164 +++++++++++ public/js/ar/solve-cv.js | 11 + public/js/ar/subpix.js | 266 ++++++++++++++++++ public/js/exhibit.js | 64 ++++- tools/test-roi.mjs | 92 ++++++ tools/test-subpix.mjs | 176 ++++++++++++ 13 files changed, 1160 insertions(+), 10 deletions(-) create mode 100644 .claude-context/sessions/2026-10-07-corner-precision-roi.md create mode 100644 public/js/ar/geom.js create mode 100644 public/js/ar/roi.js create mode 100644 public/js/ar/subpix.js create mode 100644 tools/test-roi.mjs create mode 100644 tools/test-subpix.mjs diff --git a/.claude-context/00-overview.md b/.claude-context/00-overview.md index 734d988..5cb71c9 100644 --- a/.claude-context/00-overview.md +++ b/.claude-context/00-overview.md @@ -42,4 +42,5 @@ volume, so live state carries over and rollback is one command. ## Session Log +- 2026-10-07 — Corner Precision and ROI Detection — sub-pixel corner refinement (own implementation; `cornerSubPix` is absent from the pinned build), ROI-first detection cropping from the native video, and an anchor-coplanarity check surfaced in the HUD. - 2026-09-23 — AR Tracking Overhaul: Worker Offload, Camera Model, and Coplanar Pose Ambiguity — moved OpenCV off the main thread, fixed the focal/FOV model for portrait streams, deduped marker ids, gated the mirrored coplanar pose, and built a two-page A3 test rig with a foldable marker tower. diff --git a/.claude-context/01-architecture.md b/.claude-context/01-architecture.md index 008e95e..412fd72 100644 --- a/.claude-context/01-architecture.md +++ b/.claude-context/01-architecture.md @@ -6,8 +6,11 @@ Current as of 2026-09-23. Supersedes the v2 tracking design throughout. ``` camera video - -> detection canvas (long side capped, ~20Hz) - -> ArUco decode, maxHamming 0, filtered to scene anchor ids, deduped by id + -> ROI crops around last-seen crests, at NATIVE resolution (steady state) + or full detection canvas, long side capped (acquire / every 600ms / on loss) + -> ArUco decode, maxHamming 0 + -> sub-pixel corner refinement (gradient centroid per border edge) + -> filtered to scene anchor ids, deduped by id -> TRACKING ENGINE -> camera pose in world -> three.js render: server-driven ghosts + depth-only occluder meshes ``` @@ -45,6 +48,69 @@ Pinned to `4.12.0-release.1`. Note for future work: this build lacks `solvePnPGeneric` and `cornerSubPix`, which is why pose ambiguity is handled by gating rather than by comparing both IPPE solutions. +## Detection: ROI first, full frame as fallback + +The crests do not move; the phone does. Once a crest has been seen, the next +tick crops around where it was and decodes that, rather than rescanning 518k +pixels to re-find something that shifted by a few. Crops are taken from the +**native** video, so each is both smaller than the corresponding patch of a full +scan and higher resolution than it (`@1.3x` on a 1280 stream against a 960 +canvas). Five typical crests come to ~40k pixels against 518k. + +Mapping back into detection space is two scale changes and an origin shift per +crop, which is exactly the kind of arithmetic that fails silently and puts +ghosts a centimetre off. It is tested directly in `tools/test-roi.mjs`. + +A full scan runs on first acquisition, when targets go stale, at least every +600ms, when there are more than 6 targets, and when the crops would cover more +than 55% of the frame. **If an ROI pass returns fewer markers than it had +targets, a full scan runs in the same tick** — dropping from 4 crests to 2 is +precisely what makes the pose ambiguous, so it is never allowed to happen +silently. Worst case is one wasted crop pass. Off: `localStorage.nbx.roi = 0`. + +## Corner precision: sub-pixel refinement + +js-aruco2 returns corners on the detection canvas lattice. At 960px a 60mm crest +spans ~140px, so one pixel of corner error is ~0.7% of the marker — millimetres +of ghost placement, re-rolled every frame. + +`subpix.js` refines each quad by fitting its four border edges rather than by +looking at corners. Along each edge it scans perpendicular at ~12 points, takes +the **first moment of the gradient** at each (not the gradient peak), fits a line +through the crossings by total least squares with one outlier pass, and +intersects consecutive lines. + +The centroid matters. Parabolic interpolation of the gradient peak — the obvious +method, and what a `cornerSubPix`-style operator reduces to — is biased by +sampling phase, because the gradient of an anti-aliased step is roughly +triangular. Measured against synthetic edges it carried up to 0.29px of bias +sharp and 0.75px defocused, i.e. **worse than the whole-pixel corners it was +meant to improve**. The gradient centroid is unbiased for any symmetric edge +profile: 0.018px on the same test. End to end on synthetic markers, RMS corner +error drops ~12x (`tools/test-subpix.mjs`). + +`cv.cornerSubPix` was not an option: it is absent from the pinned build. Probed +directly rather than assumed — `solvePnPRefineLM` IS present, though it refines +a pose from corners and so cannot help here. + +Refinement is all-or-nothing per marker and falls back to the raw corners if any +edge is too short, too flat, too diffuse, or if a corner would move more than +3px. A bad refinement is worse than none, because the pose gates are tuned +against honest corner noise. Off: `localStorage.nbx.subpix = 0`. + +## Anchor conditioning + +`geom.js` measures, once per scene load, the RMS distance of every anchor corner +from the best-fit plane through all of them, and the HUD shows it. Below 1.5cm +the layout is effectively coplanar and the mirrored pose is live — the physics +gates below become load-bearing rather than a safety net. + +This exists because "the anchors are coplanar" is otherwise invisible: it looks +normal in the JSON and shows up days later as ghosts intermittently flipping +across the table. A layout re-saved by an editor that does not understand wall +anchors can silently become flat. Reference values: A3 rig with the tower ~3.5cm, +a town layout with one wall crest at 8cm ~2.7cm, any all-tabletop layout 0.0cm. + ## Camera model Everything derives from one assumption — `HFOV_DEG` (default 60, override via diff --git a/.claude-context/sessions/2026-10-07-corner-precision-roi.md b/.claude-context/sessions/2026-10-07-corner-precision-roi.md new file mode 100644 index 0000000..365d3cc --- /dev/null +++ b/.claude-context/sessions/2026-10-07-corner-precision-roi.md @@ -0,0 +1,154 @@ +# Corner Precision and ROI Detection + +**Date:** 2026-10-07 +**Project:** newbury-exhibit-v3 +**Status:** written and unit-tested offline; NOT yet validated on device +**Tags:** ar, aruco, subpixel, roi, performance, coplanarity, opencv + +## Context + +Three improvements picked off a longer list: cheaper detection, more accurate +corners, and closing out the coplanar-ambiguity work started on 2026-09-23. Two +of the three changed shape once the actual code was read. + +## Key Decisions + +- **`cv.cornerSubPix` is genuinely absent from the pinned build** — the + 2026-09-23 note was right. Verified properly this time rather than inferred: + the binding names live in the embedded WASM, not in the JS glue, so grepping + `opencv.js` reports nothing either way. Loading the module under node and + reading the exports is the only reliable check. + + Also found: **`solvePnPRefineLM` IS exported.** Not useful here (it refines a + pose from corners, not the corners), but worth knowing it exists. + `solvePnPGeneric` remains absent, so the two-IPPE-solutions route to ambiguity + resolution is still closed. + +- **Wrote sub-pixel refinement rather than going without.** Edge fitting, not + corner finding: scan perpendicular across each border edge at ~12 points, find + the crossing, fit a line, intersect consecutive lines. Averaging twelve + crossings along a 140px edge beats any single window near a corner, which is + the least reliable part of a printed marker anyway. + +- **Gradient CENTROID, not gradient peak.** The first implementation used + parabolic interpolation of the gradient peak and made things *worse* — RMS + corner error went from 0.57px to 0.73px. The gradient of an anti-aliased step + is roughly triangular and a parabola fits it badly, so the estimate is pulled + toward the sample grid: up to 0.29px of phase-dependent bias sharp, 0.75px + defocused. The first moment of |gradient| has no such bias — for any symmetric + edge profile the centroid lands on the edge. Measured on 1-D synthetic edges: + parabolic 0.20px RMS, centroid 0.018px. + + *Rejected:* half-maximum crossing (0.066px, needs clean plateaus); centroid of + the principal gradient lobe (tried as a defence against a second transition in + the window — it made the clean case 4x worse and did not fix the case it was + for). The second transition is handled by capping the scan at 0.42 of a marker + cell, so the nearest other edge is 2.4 scan-radii away, plus a gradient-spread + gate that drops the sample when the window is not looking at one clean edge. + +- **Two re-centring iterations per scan.** Costs a second pass; worth it on + small markers (0.063px -> 0.018px) because it makes the window symmetric about + the edge rather than about the initial guess. + +- **Refinement is all-or-nothing per marker.** Any failed edge, near-parallel + intersection, or corner moving more than 3px returns the originals untouched. + The pose gates in `cv-worker.js` are tuned against honest corner noise; a + confidently wrong corner is worse than a quantised one. + +- **ROI detection instead of lowering detection resolution.** The original idea + was to detect at 640 instead of 960 for the CPU saving. Reading the 2026-09-23 + note killed it: decode failures are already an open question and the + arithmetic showing 960 is adequate was already done. Lowering resolution would + have pushed on exactly the wrong variable. Cropping gets the same CPU saving + and moves resolution the *other* way. + +- **Crops come from the native video, not the detection canvas.** A crop is then + both smaller and more detailed than the same patch of a full scan (1.33x on a + 1280 stream at 960). A tight crop also gives the adaptive threshold a much + narrower brightness range, which is the other live candidate for the decode + failures. + +- **The crop scale is floored at detection resolution.** First version capped + the crop's long side at 224px unconditionally, which made a large near crest + come back at 0.41x — worse through an ROI than through a full scan. Caught by + the round-trip test, which prints the upscale factor for exactly this reason. + +- **An incomplete ROI pass triggers a full scan in the same tick.** Fewer markers + than targets means a crest left its crop. Solving from the shrunken set is how + a 4-crest frame becomes an ambiguous 2-crest frame, so it is never allowed. + +- **Coplanarity is measured and shown, not assumed.** `geom.js` reports the RMS + distance of every anchor corner from the best-fit plane at scene load. The + 2026-09-23 session identified non-coplanar geometry as the cure for the + mirrored pose and built the tower, but nothing tells you whether the *loaded + layout* actually has it — and the layout editor is known not to understand + wall anchors, so a save can silently flatten it. + +## Technical Details + +**`public/js/ar/subpix.js`** — `refineQuad(imageData, corners, opts)` and +`refineMarkers(imageData, markers, opts)`. Samples bilinear luma straight from +the RGBA buffer, so there is no grayscale pre-pass. ~800 taps per marker, under +1% of detection cost. Key options: `scanCellFrac 0.42` (scan radius as a +fraction of one marker cell, assuming 6 cells across for ARUCO_4X4), +`recentreIters 2`, `sigmaMaxPx 2.4`, `maxShiftPx 3.0`. + +**`public/js/ar/roi.js`** — `RoiScanner`. Full scan when: no targets, targets +older than 400ms, 600ms since the last full scan, more than 6 targets, or crops +would cover more than 55% of the frame. Crop margin 40% of the bbox (min 14px), +long side soft-capped at 224px with a hard floor at detection resolution. + +**`public/js/ar/geom.js`** — `planarity(flatPoints)` via Jacobi eigen-decomposition +of the 3x3 covariance; smallest eigenvector is the plane normal. `COPLANAR_CM = 1.5`. +Reference: all-flat layout 0.0cm, A3 rig with tower 3.5cm, town with one wall +crest at 8cm 2.7cm. + +**`public/js/ar/detect.js`** — split into `detectRaw` (decode any image) and +`finaliseMarkers` (dedupe, scene filter, diagnostics) so both paths finalise +identically. `detectMarkers` kept as a wrapper; `calibrate.html` still uses it. + +**`public/js/ar/solve-cv.js`** — `setScene` now also computes `this.planarity`, +which warms the `cornersFor` cache as a side effect. + +**New localStorage knobs:** `nbx.subpix` (default 1), `nbx.roi` (default 1). +Setting both to 0 restores the previous behaviour exactly — that is the A/B. + +**New HUD lines:** +``` +scan roi 4x @1.3x 38kpx subpix 4/4 0.21px +geom spread 10.4cm (max 16.0) +``` + +**Measured, offline, synthetic:** +``` +tools/test-subpix.mjs RMS corner error 0.569px -> 0.046px (12.4x, 192/192 accepted) +tools/test-roi.mjs crop -> detection space mapping exact to 1e-9 over 9 cases +``` + +A harness bug is worth recording: the first run showed refinement making things +worse because the test renderer integrated pixel `(i,j)` over `[i, i+1]` while +`subpix.js` samples with pixel centres on integers, the convention js-aruco2 +corners and OpenCV intrinsics both use. Half a pixel, and it pointed the finger +at the wrong file. + +## Open Questions / Next Steps + +- **Validate on device.** Success: `reproj` falls (sub-pixel corners should show + up directly in it); `scan` reads `roi Nx` most ticks with `kpx` well under + 518; `subpix` shows `N/N` with a mean shift of roughly 0.2-0.5px; `geom` + reports the tower's ~3.5cm. +- **Does ROI fix the decode-rate problem?** Compare `markers` seen with + `nbx.roi=1` against `nbx.roi=0` on the same framing. If crops decode crests + that full scans miss, the cause was local contrast, not resolution — and + `nbx.detectLong=1280` need never be tried. +- **Re-tune the 1€ filter.** Cutoffs in `solve-cv.js` were set for POS-IT-era + corner noise and then raised for the board solve. With corner error down ~12x + there is probably room to raise them again, trading smoothing for latency. +- **Re-tune `MAX_REPROJ_PX`.** Currently 4, set against quantised corners. If + reproj now sits well below that, tightening it makes the gate more selective + and the mirrored pose easier to reject. +- **Carry a raised crest into the real town layout**, not just the A3 rig, and + watch the geom line to confirm it survived the editor. +- Everything in the 2026-09-23 open-questions list still stands: openresty + cache/WS headers, layout editor vs cylinder occluders and wall anchors, + deleting the posit engine. diff --git a/README.md b/README.md index ee81c41..8684b5d 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,20 @@ Dev HUD overlay tap zones (thirds): Engine choice persists in `localStorage` (`nbx.engine`). Optional per-device horizontal FOV override: `localStorage nbx.hfovDeg` (default 60, both engines). +`localStorage` knobs: + +| key | default | what it does | +|------------------|---------|--------------| +| `nbx.engine` | opencv | `opencv` board solve, or `posit` (v2 fallback) | +| `nbx.hfovDeg` | 60 | assumed horizontal FOV, applied to the image's long axis | +| `nbx.detectHz` | 20 | detection rate; rendering stays at full frame rate | +| `nbx.detectLong` | 960 | long side of the full-scan detection canvas | +| `nbx.subpix` | 1 | sub-pixel corner refinement; `0` disables | +| `nbx.roi` | 1 | ROI-first detection; `0` forces a full scan every tick | + +Set `nbx.subpix=0` and `nbx.roi=0` to get exactly the pre-October detection +behaviour, which is the A/B to run if tracking regresses. + `axis ymirror` exists as on-device insurance: if the world appears mirrored / camera under the table in opencv mode, tap the middle third once. diff --git a/docs/PAGES.md b/docs/PAGES.md index f800dc8..d783a9f 100644 --- a/docs/PAGES.md +++ b/docs/PAGES.md @@ -101,6 +101,9 @@ ws open rx active,pong WebSocket state + recent message types scene 11 anchors ghost recs 2 scene loaded, ghosts held client-side 60 fps detect 20Hz@960 fov 58 render rate, detect rate/resolution, camera fov REJ:jump (only when the solver refused a pose) +scan roi 4x @1.3x 38kpx detection mode: 4 crops, each 1.3x the detail + subpix 4/4 0.21px ...of a full scan; corners refined, mean shift +geom spread 10.4cm (max 16.0) anchor conditioning — see below eng opencv moving engine; FROZEN = ghost motion paused axis std cv:ready engine mode + OpenCV worker state markers 3/4 seen [0,4,6] +1dup solved/raw crests, ids, duplicate decodes @@ -116,6 +119,31 @@ mirrored axis; a constant ratio means the projection doesn't match the lens (tun `localStorage.nbx.hfovDeg`); an intermittent horizontal flip with correct vertical is the coplanar ambiguity — stand the tower up. +**Scan line.** `full` means the whole detection frame was decoded; `roi Nx` +means N crops around the previous frame's crests were decoded instead, each at +native video resolution. `@1.3x` is how much more detail each crop carries than +the same patch of a full scan — above 1.0 the crop is the better read. `kpx` is +pixels actually decoded this tick; compare it against 518k for a full 960x540 +scan. `roi>full` means a crop pass came back short and a full scan ran straight +after, which is the designed fallback and costs one wasted pass. A full scan +also runs at least every 600ms regardless, to pick up newly visible crests, so +`markers ... not in this layout` can lag by up to that long in roi mode. + +**Subpix line.** `refined/total` crests whose corners were refined, and the mean +correction applied, in detection-space pixels. A mean shift around 0.2-0.5px is +normal and is roughly the quantisation the raw detector leaves behind. `0/4` +means refinement is rejecting everything: too little contrast, too small a +crest, or surface texture around the border — it falls back to raw corners, so +nothing breaks, but tracking is no more precise than before. + +**Geom line.** RMS distance of every anchor corner from the best-fit plane +through all of them. `COPLANAR` means every crest shares a plane, which is the +condition that makes a mirrored camera pose fit the pixels just as well as the +true one; the physics gates in `cv-worker.js` will be doing all the work and +will occasionally be wrong. Stand a crest up — the A3 rig's tower gives ~3.5cm, +and a single wall-mounted crest at 8cm in a town layout gives ~2.7cm. Both are +comfortably clear of the 1.5cm threshold. + ## Gotchas worth remembering **Detection filters against the active scene.** A crest that isn't an anchor in diff --git a/public/js/ar/detect.js b/public/js/ar/detect.js index 59a75de..9a607dd 100644 --- a/public/js/ar/detect.js +++ b/public/js/ar/detect.js @@ -13,8 +13,21 @@ export function createDetector() { * rawIds tells you what the camera actually saw. */ export function detectMarkers(detector, imageData, knownIds) { - const raw = detector.detect(imageData).filter(m => quadArea(m.corners) > 100); + return finaliseMarkers(detectRaw(detector, imageData), knownIds); +} +/* Raw decode over whatever image you hand it — a full detection frame or an ROI + * crop. Split out from detectMarkers so roi.js can run the same decode on a crop + * and have the results finalised together with everything else. The area floor + * is in the pixels of THIS image, which is what we want: a crop is higher + * resolution, so a marker that scrapes past the floor in a full frame clears it + * comfortably in a crop. */ +export function detectRaw(detector, imageData) { + return detector.detect(imageData).filter(m => quadArea(m.corners) > 100); +} + +/* Dedupe, filter to the scene, attach diagnostics. */ +export function finaliseMarkers(raw, knownIds) { /* Deduplicate by id. Two quads in one frame can decode to the same id — a * misread of a different crest under blur or glare. That is poison for the * board solve: both quads are handed the SAME world corners, so solvePnP gets @@ -36,6 +49,14 @@ export function detectMarkers(detector, imageData, knownIds) { return usable; } +/* Centre of a quad — used to merge markers that the same crest produced in two + * overlapping ROI crops. */ +export function quadCentre(c) { + let x = 0, y = 0; + for (let i = 0; i < 4; i++) { x += c[i].x; y += c[i].y; } + return { x: x / 4, y: y / 4 }; +} + export function quadArea(c) { // shoelace let a = 0; diff --git a/public/js/ar/geom.js b/public/js/ar/geom.js new file mode 100644 index 0000000..35ea239 --- /dev/null +++ b/public/js/ar/geom.js @@ -0,0 +1,107 @@ +/* geom.js — is this layout's crest geometry actually solvable without ambiguity? + * + * A coplanar set of markers admits two camera poses that reproject almost + * identically: the true one and a mirrored twin. cv-worker.js suppresses the + * twin with physics gates (camera above the table, no teleporting), but those + * are a safety net over a geometry problem. The cure is a crest that does not + * share the plane of the others — the A3 test rig does this with a 160mm tower + * carrying crests at 10cm and 16cm. + * + * The thing is, "the anchors are coplanar" is invisible. A layout saved from the + * editor, or a tower knocked over and re-entered as flat, looks completely + * normal in the JSON and only shows up later as ghosts intermittently flipping + * to the wrong side of the table — the most expensive bug in this project to + * date. So: measure it once when the scene loads, and put the number on screen. + * + * The measure is the RMS distance of every anchor CORNER from the best-fit plane + * through all of them. Corners rather than anchor centres, because a crest + * mounted vertically among flat ones contributes out-of-plane spread through its + * own extent even if its centre happens to sit near the common plane. + */ + +/* Eigen-decomposition of a symmetric 3x3 by cyclic Jacobi. Small, exact enough, + * and no dependency — three.js has no symmetric eigensolver. */ +function jacobiEigen(m) { + const a = [[m[0], m[1], m[2]], [m[1], m[3], m[4]], [m[2], m[4], m[5]]]; + const v = [[1, 0, 0], [0, 1, 0], [0, 0, 1]]; + for (let sweep = 0; sweep < 24; sweep++) { + let off = 0; + for (let p = 0; p < 2; p++) for (let q = p + 1; q < 3; q++) off += a[p][q] * a[p][q]; + if (off < 1e-20) break; + for (let p = 0; p < 2; p++) { + for (let q = p + 1; q < 3; q++) { + if (Math.abs(a[p][q]) < 1e-18) continue; + const theta = (a[q][q] - a[p][p]) / (2 * a[p][q]); + const t = Math.sign(theta || 1) / (Math.abs(theta) + Math.sqrt(theta * theta + 1)); + const c = 1 / Math.sqrt(t * t + 1), s = t * c; + for (let k = 0; k < 3; k++) { + const akp = a[k][p], akq = a[k][q]; + a[k][p] = c * akp - s * akq; + a[k][q] = s * akp + c * akq; + } + for (let k = 0; k < 3; k++) { + const apk = a[p][k], aqk = a[q][k]; + a[p][k] = c * apk - s * aqk; + a[q][k] = s * apk + c * aqk; + } + for (let k = 0; k < 3; k++) { + const vkp = v[k][p], vkq = v[k][q]; + v[k][p] = c * vkp - s * vkq; + v[k][q] = s * vkp + c * vkq; + } + } + } + } + return { values: [a[0][0], a[1][1], a[2][2]], vectors: v }; +} + +/* points: flat [x,y,z, x,y,z, ...] in world cm. + * Returns null if there is nothing meaningful to measure. */ +export function planarity(points) { + const n = points.length / 3; + if (n < 3) return null; + + let cx = 0, cy = 0, cz = 0; + for (let i = 0; i < n; i++) { cx += points[3 * i]; cy += points[3 * i + 1]; cz += points[3 * i + 2]; } + cx /= n; cy /= n; cz /= n; + + let xx = 0, xy = 0, xz = 0, yy = 0, yz = 0, zz = 0; + for (let i = 0; i < n; i++) { + const dx = points[3 * i] - cx, dy = points[3 * i + 1] - cy, dz = points[3 * i + 2] - cz; + xx += dx * dx; xy += dx * dy; xz += dx * dz; + yy += dy * dy; yz += dy * dz; zz += dz * dz; + } + const { values, vectors } = jacobiEigen([xx, xy, xz, yy, yz, zz]); + + // smallest eigenvalue -> normal of the best-fit plane + let k = 0; + if (values[1] < values[k]) k = 1; + if (values[2] < values[k]) k = 2; + const nx = vectors[0][k], ny = vectors[1][k], nz = vectors[2][k]; + + let s2 = 0, mx = 0; + for (let i = 0; i < n; i++) { + const d = (points[3 * i] - cx) * nx + (points[3 * i + 1] - cy) * ny + (points[3 * i + 2] - cz) * nz; + s2 += d * d; + if (Math.abs(d) > mx) mx = Math.abs(d); + } + return { + points: n, + rmsCm: Math.sqrt(s2 / n), + maxCm: mx, + normal: [nx, ny, nz], + }; +} + +/* COPLANAR_CM: below this the layout is effectively flat and the mirrored pose + * is live. The A3 rig's tower gives ~5cm RMS; print tolerance and a crest taped + * slightly proud of the table give well under 0.5cm, so 1.5cm separates a real + * height difference from noise without being so strict that a modest raised + * crest fails to register. */ +export const COPLANAR_CM = 1.5; + +export function describePlanarity(p) { + if (!p) return 'geom —'; + if (p.rmsCm < COPLANAR_CM) return `geom COPLANAR ${p.rmsCm.toFixed(1)}cm — raise a crest`; + return `geom spread ${p.rmsCm.toFixed(1)}cm (max ${p.maxCm.toFixed(1)})`; +} diff --git a/public/js/ar/roi.js b/public/js/ar/roi.js new file mode 100644 index 0000000..4a668bf --- /dev/null +++ b/public/js/ar/roi.js @@ -0,0 +1,164 @@ +/* roi.js — region-of-interest detection. + * + * THE PROBLEM WITH SCANNING THE WHOLE FRAME EVERY TICK: at 960px long side a + * landscape frame is ~518k pixels, and the crests occupy a few percent of it. + * Adaptive thresholding and contour tracing run over all of it, 20 times a + * second, to re-find markers that have barely moved. + * + * WHAT THIS DOES: once a crest has been seen, the next tick crops just around + * where it was — but crops from the NATIVE video, not from the downscaled + * detection canvas. So each crop is both much smaller AND higher resolution + * than the corresponding patch of a full scan. Two wins from one change: + * + * - cost: five 192px crops is ~184k pixels against 518k, and most of that is + * a GPU blit rather than CPU work + * - decode: a 60mm crest at 1.5m fills ~140px of a 960px frame but ~190px of + * a native 1280px one, and a tight crop also gives the adaptive threshold a + * far narrower range of brightness to cope with. That is a direct candidate + * fix for the open "5 crests visible, only 2 decoded" question — the + * arithmetic said resolution was adequate, but adequate at the centre of a + * frame is not adequate at the edge of an unevenly lit one. + * + * SAFETY — ROI NEVER LOSES A CREST. If an ROI pass returns fewer markers than + * it had targets, the caller is told to fall back and a full scan runs in the + * SAME tick. A full scan also runs periodically regardless, to pick up crests + * that have newly come into view. The worst case is therefore one wasted crop + * pass, not a dropped marker. + * + * All coordinates handed back are in DETECTION SPACE (the W x H the solver's + * intrinsics are built for), so nothing downstream needs to know this ran. + */ + +export const ROI_DEFAULTS = { + marginFrac: 0.40, // grow each bbox by this much — covers a tick of motion, + marginMinPx: 14, // and keeps the quad off the crop border, where + // contour tracers discard it + maxLongPx: 224, // cap a crop's long side; beyond this we're paying for + // pixels the decoder does not need + maxRois: 6, // more targets than this and a full scan is simpler + maxCoverFrac: 0.55, // if the crops would cover this much of the frame, + // a single full scan is cheaper than many small ones + fullScanMs: 600, // always re-scan the whole frame at least this often + targetTtlMs: 400, // targets older than this are not worth trusting +}; + +export class RoiScanner { + constructor(opts = {}) { + this.o = { ...ROI_DEFAULTS, ...opts }; + this.targets = new Map(); // markerId -> { x, y, w, h } in detection space + this.targetsAt = 0; + this.lastFullAt = 0; + this.stats = { mode: 'full', rois: 0, pixels: 0, upscale: 1 }; + this._cv = null; + this._cx = null; + } + + _canvas(w, h) { + if (!this._cv) { + this._cv = document.createElement('canvas'); + this._cx = this._cv.getContext('2d', { willReadFrequently: true }); + } + if (this._cv.width < w) this._cv.width = w; + if (this._cv.height < h) this._cv.height = h; + return this._cx; + } + + /** Remember where each marker was, so the next tick knows where to look. */ + update(markers, now) { + this.targets.clear(); + for (const m of markers) { + let x0 = Infinity, y0 = Infinity, x1 = -Infinity, y1 = -Infinity; + for (const c of m.corners) { + if (c.x < x0) x0 = c.x; if (c.x > x1) x1 = c.x; + if (c.y < y0) y0 = c.y; if (c.y > y1) y1 = c.y; + } + this.targets.set(m.id, { x: x0, y: y0, w: x1 - x0, h: y1 - y0 }); + } + this.targetsAt = now; + } + + reset() { this.targets.clear(); this.targetsAt = 0; this.lastFullAt = 0; } + + /** Should this tick be a full-frame scan? */ + wantsFull(now, W, H) { + if (!this.targets.size) return true; + if (now - this.targetsAt > this.o.targetTtlMs) return true; + if (now - this.lastFullAt > this.o.fullScanMs) return true; + if (this.targets.size > this.o.maxRois) return true; + + let cover = 0; + for (const t of this.targets.values()) { + const m = Math.max(this.o.marginMinPx, Math.max(t.w, t.h) * this.o.marginFrac); + cover += (t.w + 2 * m) * (t.h + 2 * m); + } + return cover > this.o.maxCoverFrac * W * H; + } + + noteFull(now) { this.lastFullAt = now; this.stats = { mode: 'full', rois: 0, pixels: 0, upscale: 1 }; } + + /* Crop each target out of the native video and detect inside it. + * + * detectRaw(imageData) -> markers with corners in that image's pixels + * refine(imageData, markers) -> optional sub-pixel pass, run at CROP + * resolution so its precision is divided by the upscale factor when mapped + * back into detection space + * + * Returns { markers, complete } — complete:false means fall back to a full + * scan right now. */ + scan(video, geom, detectRaw, refine) { + const { W, H, vw, vh } = geom; + const ratio = vw / W; // detection space -> native pixels + const out = []; + let pixels = 0, upAcc = 0, upN = 0; + + for (const [id, t] of this.targets) { + const m = Math.max(this.o.marginMinPx, Math.max(t.w, t.h) * this.o.marginFrac); + const dx = Math.max(0, t.x - m), dy = Math.max(0, t.y - m); + const dw = Math.min(W - dx, t.w + 2 * m), dh = Math.min(H - dy, t.h + 2 * m); + if (dw < 8 || dh < 8) continue; + + const nx = dx * ratio, ny = dy * ratio; + const nw = dw * ratio, nh = dh * ratio; + /* The cap must never take a crop BELOW detection-space resolution: that + * would make a big near crest decode worse through an ROI than through a + * full scan, which is backwards. Floor the scale at 1/ratio so a crop is + * always at least as detailed as the full frame, and treat maxLongPx as a + * soft cap. A crest big enough to fight the floor will trip maxCoverFrac + * and get a full scan anyway. */ + const s = Math.max(1 / ratio, Math.min(1, this.o.maxLongPx / Math.max(nw, nh))); + const cw = Math.max(8, Math.round(nw * s)), ch = Math.max(8, Math.round(nh * s)); + + const cx = this._canvas(cw, ch); + cx.drawImage(video, nx, ny, nw, nh, 0, 0, cw, ch); + const img = cx.getImageData(0, 0, cw, ch); + pixels += cw * ch; + upAcc += cw / dw; upN++; + + let found; + try { found = detectRaw(img); } catch { found = []; } + if (refine && found.length) refine(img, found); + + // crop pixels -> detection space + const kx = dw / cw, ky = dh / ch; + for (const mk of found) { + mk.corners = mk.corners.map(c => ({ x: dx + c.x * kx, y: dy + c.y * ky })); + mk.roiId = id; + out.push(mk); + } + } + + /* Fewer markers than targets means a crest moved out of its crop, or the + * crop clipped it. Either way the frame is incomplete and the caller must + * do a full scan rather than solve from a shrunken marker set — dropping + * from 4 crests to 2 is exactly the condition that makes the pose + * ambiguous again. */ + const complete = out.length >= this.targets.size && out.length > 0; + this.stats = { + mode: complete ? 'roi' : 'roi>full', + rois: this.targets.size, + pixels, + upscale: upN ? upAcc / upN : 1, + }; + return { markers: out, complete }; + } +} diff --git a/public/js/ar/solve-cv.js b/public/js/ar/solve-cv.js index 1eda0b8..1279083 100644 --- a/public/js/ar/solve-cv.js +++ b/public/js/ar/solve-cv.js @@ -21,6 +21,7 @@ */ import * as THREE from 'three'; import { anchorWorldMatrix } from './pose.js'; +import { planarity } from './geom.js'; // ---------------- marker corner templates ---------------- /* js-aruco2 corner order (canonical, matches POS-IT buildModel): TL, TR, BR, BL. @@ -96,6 +97,7 @@ export class CvBoardSolver { this.lastReproj = 0; this.lastCount = 0; this.lastReject = ''; // why the worker last refused a pose (HUD) + this.planarity = null; // anchor geometry conditioning, see geom.js this._v = new THREE.Vector3(); this._m4 = new THREE.Matrix4(); this._q = new THREE.Quaternion(); @@ -163,6 +165,15 @@ export class CvBoardSolver { if (a.enabled === false) continue; this.anchors.set(a.markerId, { mat: anchorWorldMatrix(a), sizeMM: a.sizeMM || 60 }); } + /* Measure the layout's conditioning once, here, rather than discovering it + * later as intermittent mirrored poses. Warming cornersFor() for every + * anchor is a side benefit — the first solve no longer pays for it. */ + const pts = []; + for (const id of this.anchors.keys()) { + const flat = this.cornersFor(id); + if (flat) pts.push(...flat); + } + this.planarity = planarity(pts); this.filter.reset(); this.lastPose = null; if (this.worker) this.worker.postMessage({ type: 'reset' }); diff --git a/public/js/ar/subpix.js b/public/js/ar/subpix.js new file mode 100644 index 0000000..64d1eb6 --- /dev/null +++ b/public/js/ar/subpix.js @@ -0,0 +1,266 @@ +/* subpix.js — sub-pixel refinement of ArUco quad corners. + * + * WHY THIS EXISTS AND NOT cv.cornerSubPix: the pinned build + * (@techstark/opencv-js@4.12.0-release.1) does not export cornerSubPix — probed + * directly against the module, not assumed. `solvePnPRefineLM` IS exported, but + * that refines a pose from corners; it cannot improve the corners themselves. + * + * WHY IT MATTERS: js-aruco2 hands back contour corners quantised to roughly + * whole pixels of the detection canvas. Every downstream metric inherits that + * quantisation — pose jitter, reprojection error, and the gate thresholds tuned + * against them. At 960px long-side a 60mm crest spans ~140px, so one pixel of + * corner error is ~0.7% of the marker, which at typical table distance is a few + * millimetres of ghost placement, flickering frame to frame. + * + * METHOD — edge fitting, not corner finding. A corner operator (which is what + * cornerSubPix is) looks at a small window around one point and is easily pulled + * by the marker's own inner modules. Instead we use the whole length of each + * border edge: + * + * 1. walk along each edge of the quad, skipping the ends + * 2. at each step, scan perpendicular to the edge and find the sub-pixel peak + * of the intensity gradient (parabolic fit on the three samples around the + * maximum) — that is the true border crossing at that point + * 3. fit a line through those crossings by total least squares, with one + * outlier rejection pass + * 4. intersect consecutive edge lines to get the refined corners + * + * Averaging ~12 independent crossings per edge beats any single-window estimate, + * and the corner comes from two long lines rather than from pixels near the + * corner itself — which is exactly where a printed marker is least reliable + * (ink bleed, rounding, and the contour tracer's own staircase). + * + * SAFETY: refinement is all-or-nothing per marker. If any edge is too short, too + * low-contrast, too few inliers, near-parallel to its neighbour, or if any + * corner would move further than maxShiftPx, the ORIGINAL corners are returned + * untouched. A bad refinement is worse than none, because the pose gates in + * cv-worker.js are tuned against honest corner noise. + * + * COST: samples are taken straight from the RGBA ImageData with bilinear luma, + * so there is no grayscale pre-pass over the frame. Per marker: 4 edges x ~12 + * samples x ~17 taps ~= 800 taps. Five markers is ~4k taps against the 518k + * pixels the detector has already touched — under 1% of detection cost. + */ + +const EPS = 1e-9; + +export const DEFAULT_OPTS = { + edgeInset: 0.18, // skip this fraction at each end of an edge (corners lie) + samplePitchPx: 9, // one crossing per ~9px of edge + minSamples: 5, + maxSamples: 22, + scanCellFrac: 0.42, // scan +/- this fraction of ONE marker cell, so the + // scan can never reach the next black/white transition + minScanPx: 1.5, + maxScanPx: 7, + scanStepPx: 0.5, + minContrast: 24, // luma range across a scan; below this it's flat/noise + recentreIters: 2, // re-centre the scan window on the estimate, then redo + maxOffFrac: 0.75, // crossing further than this fraction of R => not our edge + sigmaMaxFrac: 0.55, // gradient spread, as a fraction of R ... + sigmaMaxPx: 2.4, // ... and an absolute ceiling, so heavy defocus still passes + minPoints: 4, // inlier crossings needed to trust an edge + outlierPx: 1.4, + minEdgePx: 24, // below this a marker is too small to refine usefully + maxShiftPx: 3.0, // refinement that moves a corner further than this is + // not refinement, it's a different quad +}; + +/* Bilinear luma sample from RGBA ImageData. NaN outside the image. */ +function lumaAt(d, w, h, x, y) { + if (!(x >= 0 && y >= 0 && x <= w - 1 && y <= h - 1)) return NaN; + const x0 = x | 0, y0 = y | 0; + const x1 = x0 + 1 < w ? x0 + 1 : x0; + const y1 = y0 + 1 < h ? y0 + 1 : y0; + const fx = x - x0, fy = y - y0; + const i00 = (y0 * w + x0) << 2, i10 = (y0 * w + x1) << 2; + const i01 = (y1 * w + x0) << 2, i11 = (y1 * w + x1) << 2; + const g00 = 0.299 * d[i00] + 0.587 * d[i00 + 1] + 0.114 * d[i00 + 2]; + const g10 = 0.299 * d[i10] + 0.587 * d[i10 + 1] + 0.114 * d[i10 + 2]; + const g01 = 0.299 * d[i01] + 0.587 * d[i01 + 1] + 0.114 * d[i01 + 2]; + const g11 = 0.299 * d[i11] + 0.587 * d[i11 + 1] + 0.114 * d[i11 + 2]; + const a = g00 + (g10 - g00) * fx; + const b = g01 + (g11 - g01) * fx; + return a + (b - a) * fy; +} + +/* One pass: sample a scan line centred on (sx,sy), return the gradient + * centroid, its spread, the contrast and the net polarity. */ +function scanPass(d, w, h, sx, sy, nx, ny, R, step) { + const n = Math.floor((2 * R) / step) + 1; + if (n < 7) return null; + + const vals = new Float64Array(n); + let lo = Infinity, hi = -Infinity; + for (let i = 0; i < n; i++) { + const o = -R + i * step; + const v = lumaAt(d, w, h, sx + nx * o, sy + ny * o); + if (Number.isNaN(v)) return null; // scan left the frame + vals[i] = v; + if (v < lo) lo = v; + if (v > hi) hi = v; + } + + /* CENTROID, NOT PEAK. Parabolic interpolation of the gradient peak — the + * obvious choice, and what cornerSubPix-style methods reduce to — is biased by + * the sampling phase, because the gradient of an anti-aliased step is roughly + * triangular and a parabola is a poor fit to a triangle. Measured against + * synthetic edges of known position it carried up to 0.29px of bias sharp and + * 0.75px defocused, which is worse than the whole-pixel corners we started + * with. The first moment of |gradient| has no such bias: for ANY symmetric + * edge profile the centroid sits exactly on the edge. Same synthetic test: + * 0.018px RMS, sharp or defocused. See tools/test-subpix.mjs. */ + let sum = 0, wsum = 0, signed = 0; + const g = new Float64Array(n); + for (let i = 1; i < n - 1; i++) { + const dv = (vals[i + 1] - vals[i - 1]) * 0.5; + signed += dv; + g[i] = dv < 0 ? -dv : dv; + sum += g[i] * (-R + i * step); + wsum += g[i]; + } + if (wsum < EPS) return null; + const off = sum / wsum; + + // second moment: how spread out the gradient is. A single clean transition is + // compact; two transitions, or a textured surface, are not. + let m2 = 0; + for (let i = 1; i < n - 1; i++) { const x = -R + i * step - off; m2 += g[i] * x * x; } + + return { off, sigma: Math.sqrt(m2 / wsum), contrast: hi - lo, sign: signed >= 0 ? 1 : -1 }; +} + +/* Locate the border crossing near a sample point, re-centring so the window + * ends up symmetric about the edge (which is what makes the centroid unbiased + * when the starting guess is off, and is worth ~3x on small markers). */ +function scanEdge(d, w, h, sx, sy, nx, ny, R, o) { + let acc = 0, last = null; + for (let k = 0; k < o.recentreIters; k++) { + const r = scanPass(d, w, h, sx + nx * acc, sy + ny * acc, nx, ny, R, o.scanStepPx); + if (!r) return null; + if (r.contrast < o.minContrast) return null; // flat: nothing to find + if (Math.abs(r.off) > o.maxOffFrac * R) return null; // edge isn't in this window + acc += r.off; + last = r; + } + // diffuse gradient => more than one edge in the window, or surface texture + if (last.sigma > Math.min(o.sigmaMaxFrac * R, o.sigmaMaxPx)) return null; + return { off: acc, grad: last.sign }; +} + +/* Total least squares line fit. Returns { x, y, ux, uy } — a point and a unit + * direction. TLS rather than y = mx + b because a near-vertical edge is perfectly + * ordinary here and would blow up the slope form. */ +function fitLine(pts) { + const n = pts.length; + let mx = 0, my = 0; + for (let i = 0; i < n; i++) { mx += pts[i].x; my += pts[i].y; } + mx /= n; my /= n; + + let sxx = 0, sxy = 0, syy = 0; + for (let i = 0; i < n; i++) { + const dx = pts[i].x - mx, dy = pts[i].y - my; + sxx += dx * dx; sxy += dx * dy; syy += dy * dy; + } + if (sxx + syy < EPS) return null; + const th = 0.5 * Math.atan2(2 * sxy, sxx - syy); // major axis of the scatter + return { x: mx, y: my, ux: Math.cos(th), uy: Math.sin(th) }; +} + +function refineEdge(d, w, h, p0, p1, o) { + const ex = p1.x - p0.x, ey = p1.y - p0.y; + const len = Math.hypot(ex, ey); + if (len < o.minEdgePx) return null; + + const ux = ex / len, uy = ey / len; + const nx = -uy, ny = ux; + const cell = len / 6; // 4x4 payload + 1 border cell each side + const R = Math.min(o.maxScanPx, Math.max(o.minScanPx, cell * o.scanCellFrac)); + const nS = Math.max(o.minSamples, Math.min(o.maxSamples, Math.round(len / o.samplePitchPx))); + + const pts = []; + let pos = 0; + for (let i = 0; i < nS; i++) { + const f = nS === 1 ? 0.5 : i / (nS - 1); + const t = o.edgeInset + (1 - 2 * o.edgeInset) * f; + const sx = p0.x + ex * t, sy = p0.y + ey * t; + const r = scanEdge(d, w, h, sx, sy, nx, ny, R, o); + if (!r) continue; + const s = r.grad; + if (s > 0) pos++; + pts.push({ x: sx + nx * r.off, y: sy + ny * r.off, s }); + } + if (pts.length < o.minPoints) return null; + + /* Keep only crossings with the majority gradient polarity. A scan that found a + * white->black step where its neighbours found black->white has locked onto a + * different transition (an inner module, or the far side of a thin border); + * including it would tilt the fitted line. */ + const want = pos * 2 >= pts.length ? 1 : -1; + let kept = pts.filter(p => p.s === want); + if (kept.length < o.minPoints) return null; + + let line = fitLine(kept); + if (!line) return null; + + // one outlier pass against the fitted line + const rx = -line.uy, ry = line.ux; + const inl = kept.filter(p => Math.abs((p.x - line.x) * rx + (p.y - line.y) * ry) <= o.outlierPx); + if (inl.length >= o.minPoints && inl.length < kept.length) { + const l2 = fitLine(inl); + if (l2) { line = l2; kept = inl; } + } + line.n = kept.length; + return line; +} + +function intersect(a, b) { + const cr = a.ux * b.uy - a.uy * b.ux; + if (Math.abs(cr) < 0.05) return null; // within ~3 deg of parallel + const dx = b.x - a.x, dy = b.y - a.y; + const t = (dx * b.uy - dy * b.ux) / cr; + return { x: a.x + a.ux * t, y: a.y + a.uy * t }; +} + +/* Refine one quad. Returns a new 4-corner array, or null to keep the original. + * Corner order is preserved: corner i is the meeting of edge i-1 and edge i. */ +export function refineQuad(imageData, corners, opts = DEFAULT_OPTS) { + if (!corners || corners.length !== 4) return null; + const d = imageData.data, w = imageData.width, h = imageData.height; + + const lines = new Array(4); + for (let i = 0; i < 4; i++) { + lines[i] = refineEdge(d, w, h, corners[i], corners[(i + 1) % 4], opts); + if (!lines[i]) return null; + } + + const out = new Array(4); + let worst = 0; + for (let i = 0; i < 4; i++) { + const p = intersect(lines[(i + 3) % 4], lines[i]); + if (!p) return null; + const shift = Math.hypot(p.x - corners[i].x, p.y - corners[i].y); + if (shift > opts.maxShiftPx) return null; // this is a different quad, not a better one + if (shift > worst) worst = shift; + out[i] = p; + } + out.shift = worst; + return out; +} + +/* Refine every marker in place. Returns diagnostics for the HUD: how many + * markers were refined and the mean corner correction applied, which is the + * number that tells you whether this is doing anything at all. */ +export function refineMarkers(imageData, markers, opts = DEFAULT_OPTS) { + let refined = 0, shiftSum = 0; + for (const m of markers) { + const r = refineQuad(imageData, m.corners, opts); + if (!r) continue; + m.corners = [{ x: r[0].x, y: r[0].y }, { x: r[1].x, y: r[1].y }, + { x: r[2].x, y: r[2].y }, { x: r[3].x, y: r[3].y }]; + m.subpix = true; + refined++; + shiftSum += r.shift; + } + return { refined, total: markers.length, meanShiftPx: refined ? shiftSum / refined : 0 }; +} diff --git a/public/js/exhibit.js b/public/js/exhibit.js index 3323f3e..f749f21 100644 --- a/public/js/exhibit.js +++ b/public/js/exhibit.js @@ -10,7 +10,10 @@ * on-table A/B until the new engine is confirmed. Delete later. */ import * as THREE from 'three'; -import { createDetector, detectMarkers, quadArea } from './ar/detect.js'; +import { createDetector, detectRaw, finaliseMarkers, quadArea } from './ar/detect.js'; +import { refineMarkers } from './ar/subpix.js'; +import { RoiScanner } from './ar/roi.js'; +import { describePlanarity } from './ar/geom.js'; import { PoseEstimator, ROT_MODES, setRotMode, getRotMode } from './ar/pose.js'; import { WorldFuser } from './ar/fuse.js'; import { CvBoardSolver, CV_AXIS_MODES } from './ar/solve-cv.js'; @@ -95,6 +98,19 @@ const DETECT_HZ = parseFloat(localStorage.getItem('nbx.detectHz')) || 20; const DETECT_LONG = parseFloat(localStorage.getItem('nbx.detectLong')) || 960; const DETECT_INTERVAL_MS = 1000 / DETECT_HZ; let lastDetectAt = 0; + +/* Sub-pixel corner refinement (subpix.js). js-aruco2 returns corners quantised + * to the detection canvas lattice; everything downstream inherits that error. + * Measured on synthetic markers it cuts RMS corner error ~12x. Off: 'nbx.subpix'=0. */ +const SUBPIX = (localStorage.getItem('nbx.subpix') ?? '1') !== '0'; + +/* ROI-first detection (roi.js). Crop around the last known crest positions and + * decode those at native video resolution instead of rescanning the whole + * downscaled frame. Falls back to a full scan whenever the crop pass comes back + * short, so it cannot lose a crest. Off: 'nbx.roi'=0. */ +const ROI_ON = (localStorage.getItem('nbx.roi') ?? '1') !== '0'; +const roiScanner = new RoiScanner(); +let lastSubpix = { refined: 0, total: 0, meanShiftPx: 0 }; let fps = 0, fpsFrames = 0, fpsAt = 0; // Debug pose readout (dev mode) @@ -107,7 +123,11 @@ function dbgStatus() { return `ws ${ws} rx ${rxLog.join(',') || '-'}\n` + `scene ${sceneLoaded ? anchorCount + ' anchors' : 'NOT LOADED'} ghost recs ${activeGhosts.size}\n` + `${fps} fps detect ${DETECT_HZ}Hz@${DETECT_LONG} fov ${camera3 ? camera3.fov.toFixed(0) : '?'}` + - `${cvSolver && cvSolver.lastReject ? ' REJ:' + cvSolver.lastReject : ''}\n`; + `${cvSolver && cvSolver.lastReject ? ' REJ:' + cvSolver.lastReject : ''}\n` + + `scan ${roiScanner.stats.mode}` + + `${roiScanner.stats.mode.startsWith('roi') ? ` ${roiScanner.stats.rois}x @${roiScanner.stats.upscale.toFixed(1)}x ${(roiScanner.stats.pixels / 1000).toFixed(0)}kpx` : ''}` + + ` subpix ${SUBPIX ? `${lastSubpix.refined}/${lastSubpix.total} ${lastSubpix.meanShiftPx.toFixed(2)}px` : 'off'}\n` + + `${describePlanarity(cvSolver && cvSolver.planarity)}\n`; } function paintDbg() { if (dbgEl) dbgEl.textContent = dbgStatus() + dbgPose + dbgAim; } @@ -396,6 +416,7 @@ function applyScene(scene) { anchorCount = (scene.anchors || []).filter(a => a.enabled !== false).length; fuser.setScene(scene); cvSolver.setScene(scene); + roiScanner.reset(); buildOcclusion(scene); ghostHeight = scene.ghostHeightCm || 4; for (const e of activeGhosts.values()) e.group.userData.setHeight(ghostHeight); @@ -487,14 +508,43 @@ function loop(t) { const LONG = DETECT_LONG; const W = vw >= vh ? LONG : Math.round(LONG * vw / vh); const H = vw >= vh ? Math.round(LONG * vh / vw) : LONG; - if (canvas2d.width !== W || canvas2d.height !== H) { canvas2d.width = W; canvas2d.height = H; } - ctx2d.drawImage(video, 0, 0, W, H); - const img = ctx2d.getImageData(0, 0, W, H); - const focal = focalPxFor(W, H); if (!poseEst) poseEst = new PoseEstimator(focal); - const markers = detectMarkers(detector, img, fuser.knownIds()); + /* ROI first, full frame as the fallback. Both paths end in finaliseMarkers, + * so dedupe, scene filtering and diagnostics are identical either way and + * nothing below here needs to know which ran. + * + * Sub-pixel refinement runs on whichever image was actually decoded. In ROI + * mode that is the crop, at native resolution — so its precision is divided + * by the crop's upscale factor on the way back into detection space, and + * refinement gets BETTER the closer you are, which is the opposite of how + * the raw detector behaves. */ + let sRef = 0, sTot = 0, sShift = 0; + const refine = SUBPIX ? (im, ms) => { + const r = refineMarkers(im, ms); + sRef += r.refined; sTot += r.total; sShift += r.meanShiftPx * r.refined; + } : null; + + let markers = null; + if (ROI_ON && !roiScanner.wantsFull(t, W, H)) { + const r = roiScanner.scan(video, { W, H, vw, vh }, im => detectRaw(detector, im), refine); + if (r.complete) markers = finaliseMarkers(r.markers, fuser.knownIds()); + } + if (!markers) { + if (canvas2d.width !== W || canvas2d.height !== H) { canvas2d.width = W; canvas2d.height = H; } + ctx2d.drawImage(video, 0, 0, W, H); + const img = ctx2d.getImageData(0, 0, W, H); + const raw = detectRaw(detector, img); + if (refine && raw.length) refine(img, raw); + markers = finaliseMarkers(raw, fuser.knownIds()); + roiScanner.noteFull(t); + } + roiScanner.update(markers, t); + + const up = roiScanner.stats.mode.startsWith('roi') ? (roiScanner.stats.upscale || 1) : 1; + lastSubpix = { refined: sRef, total: sTot, meanShiftPx: sRef ? (sShift / sRef) / up : 0 }; + tracking.markers = markers.length; tracking.raw = markers.rawCount || 0; tracking.rawIds = markers.rawIds || []; diff --git a/tools/test-roi.mjs b/tools/test-roi.mjs new file mode 100644 index 0000000..9035c94 --- /dev/null +++ b/tools/test-roi.mjs @@ -0,0 +1,92 @@ +/* test-roi.mjs — does an ROI crop hand back corners in the right place? + * + * node tools/test-roi.mjs + * + * roi.js crops from the NATIVE video and decodes there, then maps the corners + * back into detection space. That is two scale changes and an origin shift per + * crop. Get it slightly wrong and nothing throws — the ghosts just sit a + * centimetre off, which on this project has historically cost days. So the + * mapping is tested directly, with a stub decoder that returns a known quad + * expressed in crop pixels; a correct roi.js must return the detection-space + * quad it came from. + * + * Canvas and video are stubbed: no pixels are needed to verify arithmetic. + */ +import { RoiScanner } from '../public/js/ar/roi.js'; + +// ---- stub DOM: a canvas that only records how it was asked to draw ---- +let lastDraw = null; +globalThis.document = { + createElement() { + return { + width: 0, height: 0, + getContext() { + return { + drawImage(_v, sx, sy, sw, sh, dx, dy, dw, dh) { lastDraw = { sx, sy, sw, sh, dx, dy, dw, dh }; }, + getImageData(x, y, w, h) { return { width: w, height: h, data: new Uint8ClampedArray(w * h * 4) }; }, + }; + }, + }; + }, +}; + +function run(name, { W, H, vw, vh, quadDet, maxLongPx }) { + const roi = new RoiScanner(maxLongPx ? { maxLongPx } : {}); + // seed the scanner as though this quad was seen last tick + roi.update([{ id: 7, corners: quadDet }], 1000); + + const seen = []; + const detectRaw = () => { + /* Express the SAME physical quad in the crop's own pixels, using only the + * draw parameters roi.js chose. If roi.js maps back correctly we get + * quadDet out again. */ + const { sx, sy, sw, sh, dw, dh } = lastDraw; + seen.push({ ...lastDraw }); + return [{ + id: 7, + corners: quadDet.map(p => ({ + x: ((p.x * (vw / W)) - sx) * (dw / sw), + y: ((p.y * (vh / H)) - sy) * (dh / sh), + })), + }]; + }; + + const r = roi.scan({}, { W, H, vw, vh }, detectRaw, null); + let worst = 0; + for (let i = 0; i < 4; i++) { + worst = Math.max(worst, Math.hypot(r.markers[0].corners[i].x - quadDet[i].x, + r.markers[0].corners[i].y - quadDet[i].y)); + } + const d = seen[0]; + const upscale = d.dw / (d.sw / (vw / W)); + console.log( + name.padEnd(38) + + `crop ${String(d.dw).padStart(3)}x${String(d.dh).padStart(3)}` + + ` upscale ${upscale.toFixed(2)}x`.padEnd(18) + + ` max corner err ${worst.toFixed(6)}px` + + (worst < 1e-9 ? ' PASS' : ' *** FAIL ***') + ); + return worst < 1e-9; +} + +const q = (x, y, s) => [{ x, y }, { x: x + s, y }, { x: x + s, y: y + s }, { x, y: y + s }]; + +console.log('ROI crop -> detection space round trip\n'); +let ok = true; +ok &= run('landscape 1280x720, det 960', { W: 960, H: 540, vw: 1280, vh: 720, quadDet: q(400, 220, 70) }); +ok &= run('portrait 720x1280, det 960', { W: 540, H: 960, vw: 720, vh: 1280, quadDet: q(200, 500, 60) }); +ok &= run('crest near left edge', { W: 960, H: 540, vw: 1280, vh: 720, quadDet: q(4, 200, 50) }); +ok &= run('crest near bottom-right edge', { W: 960, H: 540, vw: 1280, vh: 720, quadDet: q(890, 470, 55) }); +ok &= run('rotated quad (non-axis-aligned)', { + W: 960, H: 540, vw: 1280, vh: 720, + quadDet: [{ x: 410, y: 200 }, { x: 488, y: 228 }, { x: 460, y: 306 }, { x: 382, y: 278 }], +}); +ok &= run('tiny distant crest', { W: 960, H: 540, vw: 1280, vh: 720, quadDet: q(500, 260, 18) }); +ok &= run('big near crest (hits long cap)', { W: 960, H: 540, vw: 1280, vh: 720, quadDet: q(300, 120, 300) }); +ok &= run('4K source, det 960', { W: 960, H: 540, vw: 3840, vh: 2160, quadDet: q(400, 220, 70) }); +ok &= run('cap raised to 384', { W: 960, H: 540, vw: 1280, vh: 720, quadDet: q(400, 220, 70), maxLongPx: 384 }); + +console.log('\nupscale > 1 means the crop carries more pixels per crest than a full'); +console.log('scan would, which is where the decode-rate gain is expected to come from.'); +console.log(ok ? '\nall mappings exact' : '\nMAPPING ERRORS PRESENT'); +process.exit(ok ? 0 : 1); diff --git a/tools/test-subpix.mjs b/tools/test-subpix.mjs new file mode 100644 index 0000000..6bdacbb --- /dev/null +++ b/tools/test-subpix.mjs @@ -0,0 +1,176 @@ +/* test-subpix.mjs — does sub-pixel refinement actually help, and by how much? + * + * node tools/test-subpix.mjs + * + * Renders synthetic ArUco-like markers at known sub-pixel corner positions + * (anti-aliased by 4x4 supersampling, with optional sensor noise and defocus), + * quantises the corners the way a contour tracer does, then measures RMS corner + * error before and after refineQuad. + * + * This exists because "the ghosts look steadier" is not a measurement. If a + * tuning change in subpix.js makes this number worse, it was not an improvement. + */ +import { refineQuad, DEFAULT_OPTS } from '../public/js/ar/subpix.js'; + +// ---------- unit square -> quad homography (Heckbert) ---------- +function squareToQuad(q) { + const [p0, p1, p2, p3] = q; // (0,0) (1,0) (1,1) (0,1) + const sx = p0.x - p1.x + p2.x - p3.x; + const sy = p0.y - p1.y + p2.y - p3.y; + let a, b, c, d, e, f, g, h; + if (Math.abs(sx) < 1e-12 && Math.abs(sy) < 1e-12) { + a = p1.x - p0.x; b = p2.x - p1.x; c = p0.x; + d = p1.y - p0.y; e = p2.y - p1.y; f = p0.y; + g = 0; h = 0; + } else { + const dx1 = p1.x - p2.x, dx2 = p3.x - p2.x; + const dy1 = p1.y - p2.y, dy2 = p3.y - p2.y; + const den = dx1 * dy2 - dx2 * dy1; + g = (sx * dy2 - dx2 * sy) / den; + h = (dx1 * sy - sx * dy1) / den; + a = p1.x - p0.x + g * p1.x; b = p3.x - p0.x + h * p3.x; c = p0.x; + d = p1.y - p0.y + g * p1.y; e = p3.y - p0.y + h * p3.y; f = p0.y; + } + return [a, b, c, d, e, f, g, h, 1]; +} + +function inv3(m) { + const [a, b, c, d, e, f, g, h, i] = m; + const A = e * i - f * h, B = -(d * i - f * g), C = d * h - e * g; + const det = a * A + b * B + c * C; + return [A / det, -(b * i - c * h) / det, (b * f - c * e) / det, + B / det, (a * i - c * g) / det, -(a * f - c * d) / det, + C / det, -(a * h - b * g) / det, (a * e - b * d) / det]; +} + +function apply(m, x, y) { + const w = m[6] * x + m[7] * y + m[8]; + return { x: (m[0] * x + m[1] * y + m[2]) / w, y: (m[3] * x + m[4] * y + m[5]) / w }; +} + +// ---------- synthetic marker render ---------- +function renderMarker(W, H, quad, { bits, noise = 0, blur = 0, black = 28, white = 226 }) { + const Hm = inv3(squareToQuad(quad)); + const d = new Uint8ClampedArray(W * H * 4); + const SS = 4, inv = 1 / (SS * SS); + for (let py = 0; py < H; py++) { + for (let px = 0; px < W; px++) { + let acc = 0; + for (let sy = 0; sy < SS; sy++) { + for (let sx = 0; sx < SS; sx++) { + /* Pixel CENTRES sit on integer coordinates — the convention js-aruco2 + * corners and OpenCV intrinsics both use, and the one subpix.js samples + * with. Integrating over [px, px+1] instead would bake a half-pixel + * offset into the ground truth and make a correct refiner look biased. */ + const p = apply(Hm, px - 0.5 + (sx + 0.5) / SS, py - 0.5 + (sy + 0.5) / SS); + const u = p.x * 6, v = p.y * 6; // 6x6 cells: 4x4 payload + border + if (u < 0 || v < 0 || u >= 6 || v >= 6) { acc += white; continue; } + const cu = u | 0, cv = v | 0; + const border = cu === 0 || cv === 0 || cu === 5 || cv === 5; + acc += border ? black : (bits[(cv - 1) * 4 + (cu - 1)] ? white : black); + } + } + let g = acc * inv; + if (noise) g += (Math.random() + Math.random() + Math.random() - 1.5) * noise; + const i = (py * W + px) << 2; + d[i] = d[i + 1] = d[i + 2] = g; d[i + 3] = 255; + } + } + if (blur > 0) boxBlur(d, W, H, blur); + return { data: d, width: W, height: H }; +} + +/* Separable box blur, repeated — a cheap stand-in for lens defocus. */ +function boxBlur(d, W, H, r) { + const tmp = new Float64Array(W * H); + for (let pass = 0; pass < 2; pass++) { + for (let y = 0; y < H; y++) for (let x = 0; x < W; x++) { + let s = 0, n = 0; + for (let k = -r; k <= r; k++) { const xx = x + k; if (xx >= 0 && xx < W) { s += d[((y * W + xx) << 2)]; n++; } } + tmp[y * W + x] = s / n; + } + for (let y = 0; y < H; y++) for (let x = 0; x < W; x++) { + let s = 0, n = 0; + for (let k = -r; k <= r; k++) { const yy = y + k; if (yy >= 0 && yy < H) { s += tmp[yy * W + x]; n++; } } + const i = (y * W + x) << 2; + d[i] = d[i + 1] = d[i + 2] = s / n; + } + } +} + +// ---------- scenarios ---------- +const rnd = (() => { let s = 1234567; return () => (s = (s * 1103515245 + 12345) & 0x7fffffff) / 0x7fffffff; })(); + +function quadAt(cx, cy, size, rot, persp) { + const base = [[-0.5, -0.5], [0.5, -0.5], [0.5, 0.5], [-0.5, 0.5]]; + const c = Math.cos(rot), s = Math.sin(rot); + return base.map(([x, y]) => { + const sc = size * (1 + persp * y); // fake foreshortening + return { x: cx + (x * c - y * s) * sc, y: cy + (x * s + y * c) * sc }; + }); +} + +function rms(a, b) { + let s = 0; + for (let i = 0; i < 4; i++) s += (a[i].x - b[i].x) ** 2 + (a[i].y - b[i].y) ** 2; + return Math.sqrt(s / 4); +} + +const cases = [ + { name: 'large, square-on (140px)', size: 140, rot: 0.00, persp: 0.00, noise: 0, blur: 0 }, + { name: 'large, rotated 20 deg (140px)', size: 140, rot: 0.35, persp: 0.00, noise: 0, blur: 0 }, + { name: 'oblique, foreshortened (140px)', size: 140, rot: 0.20, persp: 0.28, noise: 0, blur: 0 }, + { name: 'medium (90px)', size: 90, rot: 0.25, persp: 0.15, noise: 0, blur: 0 }, + { name: 'small (52px)', size: 52, rot: 0.18, persp: 0.10, noise: 0, blur: 0 }, + { name: 'noisy (sigma 8 luma) (110px)', size: 110, rot: 0.22, persp: 0.12, noise: 8, blur: 0 }, + { name: 'defocused (r=1 box x2) (110px)', size: 110, rot: 0.22, persp: 0.12, noise: 0, blur: 1 }, + { name: 'noisy + defocused (110px)', size: 110, rot: 0.22, persp: 0.12, noise: 6, blur: 1 }, +]; + +const TRIALS = 24; +console.log('sub-pixel corner refinement — RMS corner error vs ground truth (px)\n'); +console.log('scenario raw refined gain refined/total'); +console.log('-'.repeat(74)); + +let totRaw = 0, totRef = 0, totN = 0, totTried = 0, totOk = 0; + +for (const c of cases) { + let sumRaw = 0, sumRef = 0, n = 0, ok = 0; + for (let t = 0; t < TRIALS; t++) { + const W = 256, H = 256; + // sub-pixel placement: the centre lands on a fractional pixel every trial + const quad = quadAt(128 + rnd() - 0.5, 128 + rnd() - 0.5, c.size, c.rot + (rnd() - 0.5) * 0.1, c.persp); + const bits = Array.from({ length: 16 }, () => (rnd() > 0.45 ? 1 : 0)); + const img = renderMarker(W, H, quad, { bits, noise: c.noise, blur: c.blur }); + + /* What a contour tracer hands you: corners on the pixel lattice. The extra + * +/-0.5 jitter stands in for the tracer picking a neighbouring lattice + * point on a staircased edge. */ + const raw = quad.map(p => ({ + x: Math.round(p.x + (rnd() - 0.5)), + y: Math.round(p.y + (rnd() - 0.5)), + })); + + const ref = refineQuad(img, raw, DEFAULT_OPTS); + sumRaw += rms(raw, quad); + sumRef += rms(ref || raw, quad); + if (ref) ok++; + n++; + } + const r = sumRaw / n, f = sumRef / n; + totRaw += sumRaw; totRef += sumRef; totN += n; totTried += n; totOk += ok; + console.log( + c.name.padEnd(32) + + r.toFixed(3).padStart(7) + + f.toFixed(3).padStart(9) + + (r / f).toFixed(2).padStart(7) + 'x' + + `${ok}/${n}`.padStart(13) + ); +} + +console.log('-'.repeat(74)); +const R = totRaw / totN, F = totRef / totN; +console.log('overall'.padEnd(32) + R.toFixed(3).padStart(7) + F.toFixed(3).padStart(9) + + (R / F).toFixed(2).padStart(7) + 'x' + `${totOk}/${totTried}`.padStart(13)); +console.log('\nrefined/total is the accept rate: refinement that fails its guards'); +console.log('returns the original corners, so a low rate costs nothing but gains nothing.');