From ffaceaf962da9593752f7d71bbda66a3baa34c00 Mon Sep 17 00:00:00 2001 From: John O'Keefe Date: Mon, 13 Apr 2026 16:43:25 -0400 Subject: [PATCH] Add panel detection module with multi-tier fallback system Implement a comprehensive panel detection system for manga and comics with automatic fallback chain for maximum compatibility. Core detector (detector.js): - PanelDetector class with in-memory caching - Lazy-loading of OpenCV and TensorFlow.js - Validation logic to filter poor detections - Cache management to avoid re-detection OpenCV edge detection (opencv.js): - Canny edge detection for panel boundaries - Contour finding with bounding box extraction - Size and aspect ratio filtering - Reading order sorting (top-to-bottom, left-to-right) ML-based detection (coco-ssd.js): - COCO-SSD pre-trained model integration - Object detection for irregular panel layouts - Rectangular filtering for panel-like regions - Handles edge cases where edge detection fails Grid-based fallback (grid.js): - Lightweight 3x3 grid detection - Empty cell detection via alpha channel analysis - Adjacent panel merging algorithm - Always works as final fallback The detection pipeline tries OpenCV first (fast, accurate), falls back to ML detection if validation fails, and uses grid detection as ultimate baseline. --- panel-detection/coco-ssd.js | 47 +++++++++++++ panel-detection/detector.js | 122 ++++++++++++++++++++++++++++++++++ panel-detection/grid.js | 127 ++++++++++++++++++++++++++++++++++++ panel-detection/opencv.js | 61 +++++++++++++++++ 4 files changed, 357 insertions(+) create mode 100644 panel-detection/coco-ssd.js create mode 100644 panel-detection/detector.js create mode 100644 panel-detection/grid.js create mode 100644 panel-detection/opencv.js diff --git a/panel-detection/coco-ssd.js b/panel-detection/coco-ssd.js new file mode 100644 index 0000000..8acaa9b --- /dev/null +++ b/panel-detection/coco-ssd.js @@ -0,0 +1,47 @@ +// panel-detection/coco-ssd.js +// ML-based panel detection using COCO-SSD +export async function detectPanelsML(imageData, model) { + const canvas = document.createElement("canvas"); + canvas.width = imageData.width; + canvas.height = imageData.height; + const ctx = canvas.getContext("2d"); + ctx.putImageData(imageData, 0, 0); + + const predictions = await model.detect(canvas); + + const panels = []; + const imgWidth = imageData.width; + const imgHeight = imageData.height; + + for (let i = 0; i < predictions.length; i++) { + const pred = predictions[i]; + const [x, y, w, h] = pred.bbox; + const aspectRatio = w / h; + + const isRectangular = + aspectRatio > 0.3 && + aspectRatio < 5 && + w > imgWidth * 0.05 && + h > imgHeight * 0.05; + + if (isRectangular) { + panels.push({ + id: `ml-${i}`, + x: (x / imgWidth) * 100, + y: (y / imgHeight) * 100, + width: (w / imgWidth) * 100, + height: (h / imgHeight) * 100, + reading_order: i, + }); + } + } + + panels.sort((a, b) => { + const rowA = Math.floor(a.y / 20); + const rowB = Math.floor(b.y / 20); + if (rowA !== rowB) return rowA - rowB; + return a.x - b.x; + }); + + return panels.map((p, i) => ({ ...p, reading_order: i })); +} diff --git a/panel-detection/detector.js b/panel-detection/detector.js new file mode 100644 index 0000000..6585e8c --- /dev/null +++ b/panel-detection/detector.js @@ -0,0 +1,122 @@ +// panel-detection/detector.js +// Main panel detector with lazy-loaded fallback chain +export class PanelDetector { + #opencv = null; + #model = null; + #cache = new Map(); + + async detectPanels(doc, index, force = false) { + const cacheKey = `${doc.location?.pathname || ""}-${index}`; + + if (!force && this.#cache.has(cacheKey)) { + return this.#cache.get(cacheKey); + } + + const imageData = this.#extractImageData(doc); + if (!imageData) { + return { panels: [], method: "no-image", confidence: 0 }; + } + + const result = await this.#runDetectionPipeline(imageData); + this.#cache.set(cacheKey, result); + return result; + } + + #extractImageData(doc) { + const img = doc.querySelector("img") || doc.querySelector("canvas"); + if (!img) return null; + + const canvas = document.createElement("canvas"); + canvas.width = img.naturalWidth || img.width; + canvas.height = img.naturalHeight || img.height; + const ctx = canvas.getContext("2d"); + ctx.drawImage(img, 0, 0); + return ctx.getImageData(0, 0, canvas.width, canvas.height); + } + + async #runDetectionPipeline(imageData) { + const { detectPanelsOpenCV } = await import("./opencv.js"); + const { detectPanelsML } = await import("./coco-ssd.js"); + const { detectPanelsGrid } = await import("./grid.js"); + + if (!this.#opencv) { + try { + this.#opencv = await this.#loadOpenCV(); + } catch (e) { + console.warn("Failed to load OpenCV:", e); + } + } + + if (this.#opencv) { + try { + const panels = await detectPanelsOpenCV(imageData, this.#opencv); + if (this.#validatePanels(panels, imageData)) { + return { panels, method: "opencv", confidence: 0.85 }; + } + } catch (e) { + console.warn("OpenCV detection failed:", e); + } + } + + if (!this.#model) { + try { + this.#model = await this.#loadModel(); + } catch (e) { + console.warn("Failed to load ML model:", e); + } + } + + if (this.#model) { + try { + const panels = await detectPanelsML(imageData, this.#model); + if (this.#validatePanels(panels, imageData)) { + return { panels, method: "ml", confidence: 0.7 }; + } + } catch (e) { + console.warn("ML detection failed:", e); + } + } + + const panels = detectPanelsGrid(imageData); + return { panels, method: "grid", confidence: 0.4 }; + } + + #validatePanels(panels, imageData) { + if (!panels || panels.length === 0) return false; + if (panels.length > 30) return false; + + const imgArea = imageData.width * imageData.height; + let totalPanelArea = 0; + for (const panel of panels) { + const panelArea = ((panel.width * panel.height) / 10000) * imgArea; + totalPanelArea += panelArea; + } + + const coverage = totalPanelArea / imgArea; + return coverage > 0.1 && coverage < 0.95; + } + + async #loadOpenCV() { + const { default: cv } = await import("@techstark/opencv-js"); + await new Promise((resolve, reject) => { + const check = () => { + if (cv && cv.Mat) resolve(); + else if (!cv || cv.readyState === "complete") + reject(new Error("OpenCV failed to load")); + else setTimeout(check, 50); + }; + check(); + }); + return cv; + } + + async #loadModel() { + const tf = await import("@tensorflow/tfjs"); + const cocoSsd = await import("@tensorflow-models/coco-ssd"); + return await cocoSsd.load({ base: "lite_mobilenet_v2" }); + } + + clear() { + this.#cache.clear(); + } +} diff --git a/panel-detection/grid.js b/panel-detection/grid.js new file mode 100644 index 0000000..da1e401 --- /dev/null +++ b/panel-detection/grid.js @@ -0,0 +1,127 @@ +// panel-detection/grid.js +// Grid-based panel detection (lightweight fallback) +export function detectPanelsGrid(imageData, rows = 3, cols = 3) { + const panels = []; + const cellWidth = imageData.width / cols; + const cellHeight = imageData.height / rows; + + for (let y = 0; y < rows; y++) { + for (let x = 0; x < cols; x++) { + const startX = Math.floor(x * cellWidth); + const startY = Math.floor(y * cellHeight); + const cellData = extractCell( + imageData, + startX, + startY, + cellWidth, + cellHeight, + ); + + if (!isEmpty(cellData)) { + panels.push({ + id: `grid-${panels.length}`, + x: (x / cols) * 100, + y: (y / rows) * 100, + width: (1 / cols) * 100, + height: (1 / rows) * 100, + reading_order: panels.length, + }); + } + } + } + + return mergeAdjacentPanels(panels); +} +function extractCell(imageData, startX, startY, width, height) { + const w = Math.floor(width); + const h = Math.floor(height); + const cellData = new Uint8ClampedArray(w * h * 4); + + for (let y = 0; y < h; y++) { + for (let x = 0; x < w; x++) { + const srcIdx = ((startY + y) * imageData.width + (startX + x)) * 4; + const destIdx = (y * w + x) * 4; + cellData[destIdx] = imageData.data[srcIdx]; + cellData[destIdx + 1] = imageData.data[srcIdx + 1]; + cellData[destIdx + 2] = imageData.data[srcIdx + 2]; + cellData[destIdx + 3] = imageData.data[srcIdx + 3]; + } + } + + return { data: cellData, width: w, height: h }; +} +function isEmpty(cellData) { + let emptyPixels = 0; + const totalPixels = cellData.width * cellData.height; + + for (let i = 3; i < cellData.data.length; i += 4) { + if (cellData.data[i] < 10) emptyPixels++; + } + + return emptyPixels / totalPixels > 0.95; +} +function mergeAdjacentPanels(panels) { + const merged = []; + const used = new Set(); + + for (let i = 0; i < panels.length; i++) { + if (used.has(i)) continue; + + let current = { ...panels[i] }; + used.add(i); + + let changed = true; + while (changed) { + changed = false; + for (let j = i + 1; j < panels.length; j++) { + if (used.has(j)) continue; + if (isAdjacent(current, panels[j])) { + current = mergePanels(current, panels[j]); + used.add(j); + changed = true; + } + } + } + + merged.push(current); + } + + return merged; +} +function isAdjacent(p1, p2) { + const tolerance = 5; + if ( + Math.abs(p1.y - p2.y) < tolerance && + Math.abs(p1.height - p2.height) < tolerance + ) { + return ( + Math.abs(p1.x + p1.width - p2.x) < tolerance || + Math.abs(p2.x + p2.width - p1.x) < tolerance + ); + } + if ( + Math.abs(p1.x - p2.x) < tolerance && + Math.abs(p1.width - p2.width) < tolerance + ) { + return ( + Math.abs(p1.y + p1.height - p2.y) < tolerance || + Math.abs(p2.y + p2.height - p1.y) < tolerance + ); + } + return false; +} +function mergePanels(p1, p2) { + const minX = Math.min(p1.x, p2.x); + const minY = Math.min(p1.y, p2.y); + const maxX = Math.max(p1.x + p1.width, p2.x + p2.width); + const maxY = Math.max(p1.y + p1.height, p2.y + p2.height); + + return { + id: p1.id, + x: minX, + y: minY, + width: maxX - minX, + height: maxY - minY, + reading_order: Math.min(p1.reading_order, p2.reading_order), + }; +} diff --git a/panel-detection/opencv.js b/panel-detection/opencv.js new file mode 100644 index 0000000..412ccbf --- /dev/null +++ b/panel-detection/opencv.js @@ -0,0 +1,61 @@ +// panel-detection/opencv.js +// OpenCV-based edge detection for panel boundaries +export async function detectPanelsOpenCV(imageData, cv) { + const src = cv.matFromImageData(imageData); + const gray = new cv.Mat(); + const blurred = new cv.Mat(); + const edges = new cv.Mat(); + const contours = new cv.Mat(); + const hierarchy = new cv.Mat(); + + try { + cv.cvtColor(src, gray, cv.COLOR_RGBA2GRAY, 0); + cv.GaussianBlur(gray, blurred, new cv.Size(5, 5), 0, 0, cv.BORDER_DEFAULT); + cv.Canny(blurred, edges, 50, 150, 3, false); + cv.findContours( + edges, + contours, + hierarchy, + cv.RETR_EXTERNAL, + cv.CHAIN_APPROX_SIMPLE, + ); + + const panels = []; + const imgWidth = imageData.width; + const imgHeight = imageData.height; + + for (let i = 0; i < contours.size(); i++) { + const rect = cv.boundingRect(contours.get(i)); + const minSize = Math.min(imgWidth, imgHeight) * 0.08; + const aspectRatio = rect.width / rect.height; + + if (rect.width < minSize || rect.height < minSize) continue; + if (aspectRatio < 0.2 || aspectRatio > 8) continue; + + panels.push({ + id: `opencv-${i}`, + x: (rect.x / imgWidth) * 100, + y: (rect.y / imgHeight) * 100, + width: (rect.width / imgWidth) * 100, + height: (rect.height / imgHeight) * 100, + reading_order: i, + }); + } + + panels.sort((a, b) => { + const rowA = Math.floor(a.y / 20); + const rowB = Math.floor(b.y / 20); + if (rowA !== rowB) return rowA - rowB; + return a.x - b.x; + }); + + return panels.map((p, i) => ({ ...p, reading_order: i })); + } finally { + src.delete(); + gray.delete(); + blurred.delete(); + edges.delete(); + contours.delete(); + hierarchy.delete(); + } +}