From 15165d8fb1566e1a9c339f6a7531a4c7fca9763f Mon Sep 17 00:00:00 2001 From: John O'Keefe Date: Thu, 16 Apr 2026 20:04:05 -0400 Subject: [PATCH] improve(panel-detection): enhance OpenCV detection with connected components - Replace Canny edge detection with connected components analysis - Add Otsu thresholding and morphological operations for better border detection - Add 2-page spread detection and split processing - Improve panel size and aspect ratio filtering - Better preprocessing for comic/manga images with imperfect borders --- panel-detection/opencv.js | 227 +++++++++++++++++++++++++++++++------- 1 file changed, 188 insertions(+), 39 deletions(-) diff --git a/panel-detection/opencv.js b/panel-detection/opencv.js index 3bc4393..6e6c22f 100644 --- a/panel-detection/opencv.js +++ b/panel-detection/opencv.js @@ -1,62 +1,211 @@ // panel-detection/opencv.js -// OpenCV-based edge detection for panel boundaries -export async function detectPanelsOpenCV(imageData, cv) { +// OpenCV-based panel detection optimized for comic/manga panels +// Extract a portion of image data for 2-page spread splitting +function extractImageDataRegion(imageData, startX, startY, width, height) { + const result = new ImageData( + new Uint8ClampedArray(width * height * 4), + width, + height, + ); + + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + const srcIdx = ((startY + y) * imageData.width + (startX + x)) * 4; + const destIdx = (y * width + x) * 4; + result.data[destIdx] = imageData.data[srcIdx]; + result.data[destIdx + 1] = imageData.data[srcIdx + 1]; + result.data[destIdx + 2] = imageData.data[srcIdx + 2]; + result.data[destIdx + 3] = imageData.data[srcIdx + 3]; + } + } + + return result; +} +// Preprocess image to isolate comic panel borders +function preprocessComicImage(imageData, cv) { const src = cv.matFromImageData(imageData); const gray = new cv.Mat(); - const blurred = new cv.Mat(); - const edges = new cv.Mat(); - const contours = new cv.MatVector(); - const hierarchy = new cv.Mat(); + const binary = new cv.Mat(); + const processed = new cv.Mat(); try { + // Convert to grayscale cv.cvtColor(src, gray, cv.COLOR_RGBA2GRAY, 0); - cv.GaussianBlur(gray, blurred, new cv.Size(5, 5), 0, 0, cv.BORDER_DEFAULT); - cv.Canny(blurred, edges, 50, 150, 3, false); - cv.findContours( - edges, - contours, - hierarchy, - cv.RETR_EXTERNAL, - cv.CHAIN_APPROX_SIMPLE, - ); - const panels = []; + // Threshold using Otsu's method (auto-detects optimal threshold) + // THRESH_BINARY_INV makes dark borders white, light content black + cv.threshold(gray, binary, 0, 255, cv.THRESH_BINARY_INV + cv.THRESH_OTSU); + + // Morphological operations to connect gaps in imperfect borders + // Larger kernel (7x7) helps connect gaps in newspaper-style borders + const kernel = cv.getStructuringElement(cv.MORPH_RECT, new cv.Size(7, 7)); + + // Dilate to connect gaps + cv.dilate(binary, processed, kernel, new cv.Point(-1, -1), 2); + + // Erode to refine borders back to reasonable thickness + cv.erode(processed, processed, kernel, new cv.Point(-1, -1), 1); + + return { src, gray, binary, processed }; + } catch (e) { + // Clean up on error + src.delete(); + gray.delete(); + binary.delete(); + processed.delete(); + throw e; + } +} +// Detect panels in a single image (not split) +function detectPanelsInImage(imageData, cv) { + const { src, gray, binary, processed } = preprocessComicImage(imageData, cv); + + try { const imgWidth = imageData.width; const imgHeight = imageData.height; - for (let i = 0; i < contours.size(); i++) { - const rect = cv.boundingRect(contours.get(i)); - const minSize = Math.min(imgWidth, imgHeight) * 0.08; - const aspectRatio = rect.width / rect.height; + // Connected components analysis + const labels = new cv.Mat(); + const stats = new cv.Mat(); + const centroids = new cv.Mat(); - if (rect.width < minSize || rect.height < minSize) continue; - if (aspectRatio < 0.2 || aspectRatio > 8) continue; + const numComponents = cv.connectedComponentsWithStats( + processed, + labels, + stats, + centroids, + 8, // connectivity type (8 = all neighbors) + cv.CV_32S, + ); + const panels = []; + const minArea = imgWidth * imgHeight * 0.05; // Minimum 5% of image area + const minDimension = Math.min(imgWidth, imgHeight) * 0.15; // Minimum 15% of smallest dimension + + // Start from 1 to skip background component (label 0) + for (let i = 1; i < numComponents; i++) { + // Extract component statistics + const x = stats.data32S[i * 5]; // CC_STAT_LEFT + const y = stats.data32S[i * 5 + 1]; // CC_STAT_TOP + const width = stats.data32S[i * 5 + 2]; // CC_STAT_WIDTH + const height = stats.data32S[i * 5 + 3]; // CC_STAT_HEIGHT + const area = stats.data32S[i * 5 + 4]; // CC_STAT_AREA + + // Skip if too small (likely noise, text, or speech bubbles) + if (area < minArea) continue; + + // Skip if dimensions too small + if (width < minDimension || height < minDimension) continue; + + // Check aspect ratio (comic panels are typically rectangular) + const aspectRatio = width / height; + if (aspectRatio < 0.3 || aspectRatio > 5) continue; + + // Convert to percentages panels.push({ id: `opencv-${i}`, - x: (rect.x / imgWidth) * 100, - y: (rect.y / imgHeight) * 100, - width: (rect.width / imgWidth) * 100, - height: (rect.height / imgHeight) * 100, - reading_order: i, + x: (x / imgWidth) * 100, + y: (y / imgHeight) * 100, + width: (width / imgWidth) * 100, + height: (height / imgHeight) * 100, + reading_order: panels.length, }); } - console.log("[OpenCV] Detected", panels.length, "potential panels"); - panels.sort((a, b) => { - const rowA = Math.floor(a.y / 20); - const rowB = Math.floor(b.y / 20); - if (rowA !== rowB) return rowA - rowB; - return a.x - b.x; - }); + // Clean up + labels.delete(); + stats.delete(); + centroids.delete(); - return panels.map((p, i) => ({ ...p, reading_order: i })); + return panels; } finally { src.delete(); gray.delete(); - blurred.delete(); - edges.delete(); - contours.delete(); - hierarchy.delete(); + binary.delete(); + processed.delete(); } } +// Main detection function with 2-page spread handling +export async function detectPanelsOpenCV(imageData, cv) { + const imgWidth = imageData.width; + const imgHeight = imageData.height; + const aspectRatio = imgWidth / imgHeight; + + console.log( + `[OpenCV] Processing ${imgWidth}x${imgHeight}px image (aspect ratio: ${aspectRatio.toFixed(2)})`, + ); + + // Detect if this is a 2-page spread (aspect ratio > 1.3 suggests two pages) + if (aspectRatio > 1.3) { + console.log("[OpenCV] Detected 2-page spread, splitting for processing"); + + const halfWidth = Math.floor(imgWidth / 2); + + // Split into left and right pages + const leftData = extractImageDataRegion( + imageData, + 0, + 0, + halfWidth, + imgHeight, + ); + const rightData = extractImageDataRegion( + imageData, + halfWidth, + 0, + imgWidth - halfWidth, + imgHeight, + ); + + try { + const leftPanels = detectPanelsInImage(leftData, cv); + const rightPanels = detectPanelsInImage(rightData, cv); + + console.log( + `[OpenCV] Left page: ${leftPanels.length} panels, Right page: ${rightPanels.length} panels`, + ); + + // Adjust right panel coordinates (they're relative to right half) + const adjustedRightPanels = rightPanels.map((panel) => ({ + ...panel, + x: 50 + panel.x / 2, // Map 0-100% of right half to 50-100% of full width + width: panel.width / 2, + id: `opencv-right-${panel.id}`, + })); + + // Combine panels + const allPanels = [...leftPanels, ...adjustedRightPanels]; + + console.log(`[OpenCV] Total detected panels: ${allPanels.length}`); + + // Sort by reading order (left-to-right, top-to-bottom) + allPanels.sort((a, b) => { + const rowA = Math.floor(a.y / 20); + const rowB = Math.floor(b.y / 20); + if (rowA !== rowB) return rowA - rowB; + return a.x - b.x; + }); + + // Reassign reading_order + return allPanels.map((p, i) => ({ ...p, reading_order: i })); + } finally { + // Clean up image data regions + // (ImageData doesn't need explicit cleanup like OpenCV Mats) + } + } + + // Single page processing + const panels = detectPanelsInImage(imageData, cv); + + console.log(`[OpenCV] Detected ${panels.length} panels in single page`); + + // Sort by reading order + panels.sort((a, b) => { + const rowA = Math.floor(a.y / 20); + const rowB = Math.floor(b.y / 20); + if (rowA !== rowB) return rowA - rowB; + return a.x - b.x; + }); + + return panels.map((p, i) => ({ ...p, reading_order: i })); +}