toggle('draw')} disabled={!app.image}>
@@ -73,7 +72,7 @@
app.clearManual()}>Back to automatic
{/if}
{#if nothingDetected}
- Nothing detected, try adjusting the threshold or invert, or draw manually.
+ Nothing detected, try adjusting the threshold, clicking the piece, or draw manually.
{/if}
diff --git a/src/lib/app-state.svelte.ts b/src/lib/app-state.svelte.ts
index 4ce5c7d..10b394b 100644
--- a/src/lib/app-state.svelte.ts
+++ b/src/lib/app-state.svelte.ts
@@ -22,7 +22,6 @@ export class AppState {
scalePoints = $state([]);
lengthMm = $state(null);
threshold = $state(DEFAULT_THRESHOLD);
- invert = $state(false);
seed = $state(null);
marginMm = $state(-2);
mode = $state<'auto' | 'manual'>('auto');
@@ -45,7 +44,7 @@ export class AppState {
return shapeFromPolygons(cv, image.width, image.height, { outer: this.manualOuter, holes: this.manualHoles }, this.marginPx);
}
return detect(cv, image, {
- threshold: this.threshold, invert: this.invert, seed: this.seed, marginPx: this.marginPx,
+ threshold: this.threshold, seed: this.seed, marginPx: this.marginPx,
minHolePx: this.pxPerMm ? MIN_HOLE_MM * this.pxPerMm : 0,
});
});
diff --git a/src/lib/detect.ts b/src/lib/detect.ts
index 207ae6f..4a498f5 100644
--- a/src/lib/detect.ts
+++ b/src/lib/detect.ts
@@ -2,21 +2,91 @@ import type { Mat } from '@techstark/opencv-js';
import type { CV } from './opencv-loader';
import type { DetectParams, Point, RasterImage, Shape } from './types';
-/** Blurred grayscale; shared by thresholding and Otsu so both see the same pixels. */
-function toGray(cv: CV, image: RasterImage): Mat {
+/** Blurred, lighting-corrected grayscale; shared by thresholding and Otsu so both see the same pixels. */
+function toGray(cv: CV, image: RasterImage): { gray: Mat; background: number } {
const src = cv.matFromImageData(image as unknown as ImageData);
const gray = new cv.Mat();
try {
cv.cvtColor(src, gray, cv.COLOR_RGBA2GRAY);
cv.GaussianBlur(gray, gray, new cv.Size(5, 5), 0);
- return gray;
+ return { gray, background: flattenLighting(gray) };
} finally {
src.delete();
}
}
+/**
+ * Removes smooth lighting gradients (side light, vignetting) by fitting a quadratic surface to the
+ * photo's border band, which is almost always background. Returns the background brightness.
+ */
+function flattenLighting(gray: Mat): number {
+ const w = gray.cols;
+ const h = gray.rows;
+ const g = gray.data;
+ const band = Math.max(1, Math.round(Math.min(w, h) * 0.08));
+ const step = Math.max(1, Math.round(Math.min(w, h) / 400));
+ const terms = (x: number, y: number) => {
+ const u = x / w - 0.5;
+ const v = y / h - 0.5;
+ return [1, u, v, u * u, u * v, v * v];
+ };
+ const samples: { t: number[]; v: number }[] = [];
+ for (let y = 0; y < h; y += step) {
+ for (let x = 0; x < w; x += step) {
+ if (x >= band && x < w - band && y >= band && y < h - band) continue;
+ samples.push({ t: terms(x, y), v: g[y * w + x] });
+ }
+ }
+ // Keep samples near the median first, so a piece touching the border barely affects the fit.
+ const sorted = samples.map((s) => s.v).sort((a, b) => a - b);
+ const median = sorted[sorted.length >> 1];
+ const mad = samples.map((s) => Math.abs(s.v - median)).sort((a, b) => a - b)[samples.length >> 1];
+ const nearMedian = samples.filter((s) => Math.abs(s.v - median) <= 3.7 * mad + 2);
+ let coef = fitSurface(nearMedian);
+ // Second pass drops outliers (a piece touching the border) so they don't skew the fit.
+ const residuals = nearMedian.map((s) => s.v - dot(coef, s.t));
+ const sd = Math.sqrt(residuals.reduce((a, r) => a + r * r, 0) / residuals.length);
+ const inliers = nearMedian.filter((_, i) => Math.abs(residuals[i]) <= 2 * sd + 1);
+ if (inliers.length >= 6) coef = fitSurface(inliers);
+ const level = inliers.reduce((a, s) => a + s.v, 0) / inliers.length;
+
+ for (let y = 0; y < h; y++) {
+ for (let x = 0; x < w; x++) {
+ const i = y * w + x;
+ g[i] = Math.max(0, Math.min(255, Math.round(g[i] - dot(coef, terms(x, y)) + level)));
+ }
+ }
+ return level;
+}
+
+const dot = (a: number[], b: number[]) => a.reduce((sum, ai, i) => sum + ai * b[i], 0);
+
+/** Least-squares fit via the normal equations, solved with Gaussian elimination. */
+function fitSurface(samples: { t: number[]; v: number }[]): number[] {
+ const n = 6;
+ const m = Array.from({ length: n }, () => new Array(n + 1).fill(0));
+ for (const { t, v } of samples) {
+ for (let i = 0; i < n; i++) {
+ for (let j = 0; j < n; j++) m[i][j] += t[i] * t[j];
+ m[i][n] += t[i] * v;
+ }
+ }
+ for (let c = 0; c < n; c++) {
+ let pivot = c;
+ for (let r = c + 1; r < n; r++) if (Math.abs(m[r][c]) > Math.abs(m[pivot][c])) pivot = r;
+ [m[c], m[pivot]] = [m[pivot], m[c]];
+ if (Math.abs(m[c][c]) < 1e-12) continue;
+ for (let r = 0; r < n; r++) {
+ if (r === c) continue;
+ const f = m[r][c] / m[c][c];
+ for (let k = c; k <= n; k++) m[r][k] -= f * m[c][k];
+ }
+ }
+ return m.map((row, i) => (Math.abs(row[i]) < 1e-12 ? 0 : row[n] / row[i]));
+}
+
export function otsuThreshold(cv: CV, image: RasterImage): number {
- const gray = toGray(cv, image);
+ const { gray } = toGray(cv, image);
const out = new cv.Mat();
try {
return Math.round(cv.threshold(gray, out, 0, 255, cv.THRESH_BINARY | cv.THRESH_OTSU));
@@ -81,20 +151,24 @@ export function traceMask(cv: CV, mask: Mat, marginPx: number): Shape | null {
}
export function detect(cv: CV, image: RasterImage, params: DetectParams): Shape | null {
- const gray = toGray(cv, image);
+ const { gray, background } = toGray(cv, image);
const mask = new cv.Mat();
const kernel = cv.getStructuringElement(cv.MORPH_ELLIPSE, new cv.Size(5, 5));
const labels = new cv.Mat();
const stats = new cv.Mat();
const centroids = new cv.Mat();
try {
- const type = params.invert ? cv.THRESH_BINARY_INV : cv.THRESH_BINARY;
- cv.threshold(gray, mask, params.threshold, 255, type);
- cv.morphologyEx(mask, mask, cv.MORPH_OPEN, kernel);
- cv.morphologyEx(mask, mask, cv.MORPH_CLOSE, kernel);
- const count = cv.connectedComponentsWithStats(mask, labels, stats, centroids, 8, cv.CV_32S);
-
- const label = pickComponent(cv, labels, stats, count, image.width, image.height, params.seed);
+ const segment = (dark: boolean, seed: Point | null) => {
+ cv.threshold(gray, mask, params.threshold, 255, dark ? cv.THRESH_BINARY_INV : cv.THRESH_BINARY);
+ cv.morphologyEx(mask, mask, cv.MORPH_OPEN, kernel);
+ cv.morphologyEx(mask, mask, cv.MORPH_CLOSE, kernel);
+ const count = cv.connectedComponentsWithStats(mask, labels, stats, centroids, 8, cv.CV_32S);
+ return pickComponent(cv, labels, stats, count, image.width, image.height, seed);
+ };
+ const seedDark = seedIsDark(gray, params.threshold, params.seed);
+ let label = seedDark === null ? null : segment(seedDark, params.seed);
+ // A click on the background makes it "the piece", which is rejected; fall back to the automatic choice.
+ if (label === null) label = segment(background > params.threshold, null);
if (label === null) return null;
const lab = labels.data32S;
@@ -173,3 +247,12 @@ function fillSmallHoles(cv: CV, mask: Mat, minHolePx: number) {
hierarchy.delete();
}
}
+
+/** The clicked pixel's side of the threshold is the piece's; null without a usable click. */
+function seedIsDark(gray: Mat, threshold: number, seed: Point | null): boolean | null {
+ if (!seed) return null;
+ const x = Math.round(seed.x);
+ const y = Math.round(seed.y);
+ if (x < 0 || y < 0 || x >= gray.cols || y >= gray.rows) return null;
+ return gray.ucharAt(y, x) <= threshold;
+}
diff --git a/src/lib/types.ts b/src/lib/types.ts
index 532b689..3deadbf 100644
--- a/src/lib/types.ts
+++ b/src/lib/types.ts
@@ -8,7 +8,6 @@ export type RasterImage = { width: number; height: number; data: Uint8ClampedArr
export type DetectParams = {
threshold: number;
- invert: boolean;
seed: Point | null;
/** Negative shrinks the piece (and grows holes). */
marginPx: number;
diff --git a/tests/detect.test.ts b/tests/detect.test.ts
index b6dcc10..a3a18ba 100644
--- a/tests/detect.test.ts
+++ b/tests/detect.test.ts
@@ -1,5 +1,5 @@
import { beforeAll, describe, expect, it } from 'vitest';
-import { detect } from '../src/lib/detect';
+import { detect, otsuThreshold } from '../src/lib/detect';
import { bounds } from '../src/lib/geometry';
import { loadOpenCV, type CV } from '../src/lib/opencv-loader';
import type { DetectParams } from '../src/lib/types';
@@ -11,7 +11,7 @@ beforeAll(async () => {
});
const params = (overrides: Partial = {}): DetectParams => ({
- threshold: 128, invert: false, seed: null, marginPx: 0, minHolePx: 0, ...overrides,
+ threshold: 128, seed: null, marginPx: 0, minHolePx: 0, ...overrides,
});
/** Light plate x 40..159, y 30..129 with hole r=15 at (100,80), plus a small light blob, on dark ground. */
@@ -31,24 +31,35 @@ describe('detect', () => {
expect(shape.holes).toHaveLength(1);
});
- it('handles a dark piece on a light background with invert', () => {
+ it('finds a dark piece on a light background without any setting', () => {
const inverted = grayImage(200, 160, (x, y) => 250 - (inRect(x, y, 40, 30, 160, 130) ? 220 : 30));
- expect(detect(cv, inverted, params())).toBeNull();
- const b = bounds(detect(cv, inverted, params({ invert: true }))!.outer);
+ const b = bounds(detect(cv, inverted, params())!.outer);
near(b.minX, 40); near(b.maxX, 159);
});
+ it('takes the polarity from the clicked pixel', () => {
+ const inverted = grayImage(200, 160, (x, y) => 250 - (inRect(x, y, 40, 30, 160, 130) ? 220 : 30));
+ const b = bounds(detect(cv, inverted, params({ seed: { x: 100, y: 80 } }))!.outer);
+ near(b.minX, 40); near(b.maxX, 159);
+ });
+
+ it('corrects uneven lighting across the photo', () => {
+ // Light from the right: the floor ramps 120 → 220 and the piece is 50 darker than the floor around it.
+ const sideLit = grayImage(400, 300, (x, y) => 120 + (100 * x) / 400 - (inRect(x, y, 100, 75, 300, 225) ? 50 : 0));
+ const shape = detect(cv, sideLit, params({ threshold: otsuThreshold(cv, sideLit) }))!;
+ const b = bounds(shape.outer);
+ near(b.minX, 100); near(b.maxX, 299); near(b.minY, 75); near(b.maxY, 224);
+ });
+
it('selects the clicked piece even when another piece is larger', () => {
const shape = detect(cv, plateScene, params({ seed: { x: 177, y: 146 } }))!;
const b = bounds(shape.outer);
near(b.minX, 170); near(b.maxX, 184);
});
- it('falls back to the automatic choice when the seed is on background or in a hole', () => {
- for (const seed of [{ x: 5, y: 5 }, { x: 100, y: 80 }]) {
- const b = bounds(detect(cv, plateScene, params({ seed }))!.outer);
- near(b.minX, 40);
- }
+ it('falls back to the automatic choice when the click is on the background', () => {
+ const b = bounds(detect(cv, plateScene, params({ seed: { x: 5, y: 5 } }))!.outer);
+ near(b.minX, 40);
});
it('ignores a seed outside the image', () => {
diff --git a/tests/fixtures.test.ts b/tests/fixtures.test.ts
index 8c0250d..159ee71 100644
--- a/tests/fixtures.test.ts
+++ b/tests/fixtures.test.ts
@@ -10,10 +10,9 @@ import type { Point, RasterImage } from '../src/lib/types';
type FixtureCase = {
file: string;
threshold: number | null;
- invert: boolean;
seed: Point | null;
pxPerMm: number;
- expected: { widthMm: number; heightMm: number; holes: number };
+ expected: { widthMm: number; heightMm: number; holes?: number };
toleranceMm: number;
};
@@ -34,12 +33,12 @@ describe.skipIf(cases.length === 0)('detect on real photos', () => {
it.each(cases)('$file', (c) => {
const image = readJpeg(dir + c.file);
const threshold = c.threshold ?? otsuThreshold(cv, image);
- const shape = detect(cv, image, { threshold, invert: c.invert, seed: c.seed, marginPx: 0, minHolePx: 3 * c.pxPerMm });
+ const shape = detect(cv, image, { threshold, seed: c.seed, marginPx: 0, minHolePx: 3 * c.pxPerMm });
expect(shape).not.toBeNull();
const mm = prepareForExport(shape!, c.pxPerMm)!;
const size = shapeSize(mm);
expect(Math.abs(size.width - c.expected.widthMm)).toBeLessThanOrEqual(c.toleranceMm);
expect(Math.abs(size.height - c.expected.heightMm)).toBeLessThanOrEqual(c.toleranceMm);
- expect(mm.holes).toHaveLength(c.expected.holes);
+ if (c.expected.holes !== undefined) expect(mm.holes).toHaveLength(c.expected.holes);
});
});
diff --git a/tests/fixtures/README.md b/tests/fixtures/README.md
index 3cef82f..937c6a9 100644
--- a/tests/fixtures/README.md
+++ b/tests/fixtures/README.md
@@ -16,7 +16,6 @@ To add one:
{
"file": "plywood-1.jpg",
"threshold": null,
- "invert": false,
"seed": null,
"pxPerMm": 3.21,
"expected": { "widthMm": 412, "heightMm": 287, "holes": 2 },
@@ -24,4 +23,5 @@ To add one:
}
```
-`threshold: null` means Otsu's automatic value. `seed` is `{ "x": ..., "y": ... }` or `null`.
+`threshold: null` means Otsu's automatic value. `holes` is optional; leave it out when
+background texture creates noise holes. Without a known scale, use `pxPerMm: 1` and sizes in px. `seed` is `{ "x": ..., "y": ... }` or `null`.
diff --git a/tests/fixtures/manifest.json b/tests/fixtures/manifest.json
index fe51488..ecbebb9 100644
--- a/tests/fixtures/manifest.json
+++ b/tests/fixtures/manifest.json
@@ -1 +1,10 @@
-[]
+[
+ {
+ "file": "mdf_floor.jpeg",
+ "threshold": null,
+ "seed": null,
+ "pxPerMm": 1,
+ "expected": { "widthMm": 1714, "heightMm": 675 },
+ "toleranceMm": 60
+ }
+]
diff --git a/tests/fixtures/mdf_floor.jpeg b/tests/fixtures/mdf_floor.jpeg
new file mode 100644
index 0000000..51b70e5
Binary files /dev/null and b/tests/fixtures/mdf_floor.jpeg differ