From 1d4195ba5ea651e380610986eb55c63516397a79 Mon Sep 17 00:00:00 2001 From: John O'Keefe Date: Sat, 4 Apr 2026 01:01:14 -0400 Subject: [PATCH] Add panel detection implementation plan documentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Document multi-tier detection pipeline: OpenCV → ML (COCO-SSD) → Grid → Manual Editor - Include dependency details with bundle sizes (OpenCV ~500KB, TensorFlow.js ~2MB) - Add architecture overview and fallback chain explanation --- PANEL_DETECTION_PLAN.md | 777 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 777 insertions(+) create mode 100644 PANEL_DETECTION_PLAN.md diff --git a/PANEL_DETECTION_PLAN.md b/PANEL_DETECTION_PLAN.md new file mode 100644 index 0000000..c29dd39 --- /dev/null +++ b/PANEL_DETECTION_PLAN.md @@ -0,0 +1,777 @@ +# Panel Detection Implementation Plan + +## Overview + +Multi-tier panel detection system with fallback chain: +**OpenCV → ML (COCO-SSD) → Grid → Manual Editor** + +Designed for a constantly growing library - handles any comic style without custom training. + +--- + +## Detection Pipeline + +``` +1. OpenCV Edge Detection (Primary) + ├─ Fast, lightweight (~500KB lazy-loaded) + ├─ Works on 80% of comics with clear panel borders + └─ Future-proof: works on unknown future comics + +2. ML Detection (COCO-SSD Fallback) + ├─ Pre-trained on millions of diverse images + ├─ Handles irregular layouts + └─ ~2MB (TensorFlow.js) + ~2MB (model), lazy-loaded + +3. Grid Detection (Baseline) + └─ Always works as final fallback + +4. Manual Editor (Last Resort) + └─ User manually draws panels +``` + +--- + +## Dependencies + +Add to `package.json`: + +```json +{ + "dependencies": { + "@techstark/opencv-js": "^4.12.0", + "@tensorflow/tfjs": "^4.22.0", + "@tensorflow-models/coco-ssd": "^2.2.3" + } +} +``` + +**Bundle sizes:** +- OpenCV.js: ~500KB (lazy-loaded) +- TensorFlow.js: ~2MB (lazy-loaded) +- COCO-SSD model: ~2MB (lazy-loaded, cached after first load) +- **Total: ~4.5MB** (acceptable for modern networks) + +--- + +## File Structure + +``` +web/src/reader/comic/ +├── panel-detection.service.ts [NEW] - Main detection service with fallback chain +├── panel-detection.opencv.ts [NEW] - OpenCV edge detection +├── panel-detection.ml.ts [NEW] - COCO-SSD ML detection +├── panel-detector.ts [MODIFY] - Add export for grid detection +├── panel-editor.ts [MODIFY] - Add re-detect, connect to service +├── page-cache.ts [OPTIONAL] - On-demand detection +├── background-color.ts [KEEP] +├── chapter-markers.ts [KEEP] +├── page-order.ts [KEEP] +├── page-scrubber.ts [KEEP] +└── panel-gap.ts [KEEP] +``` + +--- + +## Implementation + +### 1. Panel Detection Service (`panel-detection.service.ts`) + +Create this file in `web/src/reader/comic/`: + +```typescript +// Main panel detection service with fallback chain +// Priority: OpenCV → ML → Grid → Manual Editor + +interface DetectionResult { + panels: Panel[]; + method: "opencv" | "ml" | "grid" | "manual"; + confidence: number; +} + +interface Panel { + id: string; + x: number; + y: number; + width: number; + height: number; + reading_order: number; +} + +async function detectPanels( + imageData: ImageData, + allowManual: boolean = true +): Promise { + + // Tier 1: OpenCV Edge Detection + try { + const panels = await detectPanelsOpenCV(imageData); + if (validatePanels(panels, imageData)) { + return { panels, method: "opencv", confidence: 0.85 }; + } + } catch (e) { + console.warn("OpenCV detection failed:", e); + } + + // Tier 2: ML Detection (COCO-SSD) + try { + const panels = await detectPanelsML(imageData); + if (validatePanels(panels, imageData)) { + return { panels, method: "ml", confidence: 0.9 }; + } + } catch (e) { + console.warn("ML detection failed:", e); + } + + // Tier 3: Grid Detection (baseline) + const panels = detectPanelsGrid(imageData); + return { panels, method: "grid", confidence: 0.5 }; +} + +function validatePanels(panels: Panel[], imageData: ImageData): boolean { + // Must have at least 1 panel + if (panels.length === 0) return false; + + // Should not have too many panels (probably noise) + if (panels.length > 30) return false; + + // Panels should cover reasonable area (not all empty space) + let totalArea = panels.reduce((sum, p) => sum + (p.width * p.height), 0); + if (totalArea < 10 || totalArea > 100) return false; + + return true; +} + +// Import detection methods from other files +async function detectPanelsOpenCV(imageData: ImageData): Promise; +async function detectPanelsML(imageData: ImageData): Promise; +function detectPanelsGrid(imageData: ImageData, config?: { rows: number; cols: number }): Panel[]; + +export { detectPanels, DetectionResult, Panel }; +``` + +--- + +### 2. OpenCV Detection (`panel-detection.opencv.ts`) + +Create this file in `web/src/reader/comic/`: + +```typescript +// OpenCV.js-based edge detection for panel boundaries + +interface Panel { + id: string; + x: number; + y: number; + width: number; + height: number; + reading_order: number; +} + +let openCVLoaded = false; + +async function loadOpenCV(): Promise { + if (openCVLoaded) return; + + // OpenCV.js loads asynchronously and registers globally + await import("@techstark/opencv-js"); + + // Wait for OpenCV to be ready + return new Promise((resolve) => { + const check = () => { + if ((window as any).cv && (window as any).cv.Mat) { + openCVLoaded = true; + resolve(); + } else { + setTimeout(check, 50); + } + }; + check(); + }); +} + +async function detectPanelsOpenCV(imageData: ImageData): Promise { + await loadOpenCV(); + + const cv = (window as any).cv; + + // Create matrices from ImageData + const src = cv.matFromImageData(imageData); + const gray = new cv.Mat(); + const blurred = new cv.Mat(); + const edges = new cv.Mat(); + const contours = new cv.Mat(); + const hierarchy = new cv.Mat(); + + try { + // Convert to grayscale + cv.cvtColor(src, gray, cv.COLOR_RGBA2GRAY, 0); + + // Apply Gaussian blur to reduce noise + cv.GaussianBlur(gray, blurred, new cv.Size(5, 5), 0, 0, cv.BORDER_DEFAULT); + + // Detect edges using Canny + cv.Canny(blurred, edges, 50, 150, 3, false); + + // Find contours + cv.findContours( + edges, + contours, + hierarchy, + cv.RETR_EXTERNAL, + cv.CHAIN_APPROX_SIMPLE + ); + + // Convert contours to panels + const panels: Panel[] = []; + const imgWidth = imageData.width; + const imgHeight = imageData.height; + + for (let i = 0; i < contours.size(); i++) { + const rect = cv.boundingRect(contours.get(i)); + const aspectRatio = rect.width / rect.height; + + // Filter: reject very small or very thin contours + const minSize = Math.min(imgWidth, imgHeight) * 0.05; + if (rect.width < minSize || rect.height < minSize) continue; + if (aspectRatio < 0.1 || aspectRatio > 10) continue; + + panels.push({ + id: `opencv-panel-${i}`, + x: (rect.x / imgWidth) * 100, + y: (rect.y / imgHeight) * 100, + width: (rect.width / imgWidth) * 100, + height: (rect.height / imgHeight) * 100, + reading_order: i, + }); + } + + // Sort panels by reading order (top-left to bottom-right) + panels.sort((a, b) => { + const rowA = Math.floor(a.y / 25); + const rowB = Math.floor(b.y / 25); + if (rowA !== rowB) return rowA - rowB; + return a.x - b.x; + }); + + // Reassign reading order after sorting + panels.forEach((p, i) => (p.reading_order = i)); + + return panels; + + } finally { + // Clean up OpenCV matrices + src.delete(); + gray.delete(); + blurred.delete(); + edges.delete(); + contours.delete(); + hierarchy.delete(); + } +} + +export { detectPanelsOpenCV, loadOpenCV }; +``` + +--- + +### 3. ML Detection (`panel-detection.ml.ts`) + +Create this file in `web/src/reader/comic/`: + +```typescript +// ML-based panel detection using COCO-SSD pre-trained model + +interface Panel { + id: string; + x: number; + y: number; + width: number; + height: number; + reading_order: number; +} + +let model: any = null; +let tfLoaded = false; + +async function loadTF(): Promise { + if (tfLoaded) return; + + // Load TensorFlow.js + await import("@tensorflow/tfjs"); + tfLoaded = true; +} + +async function loadModel(): Promise { + if (model) return; + + await loadTF(); + + // Load COCO-SSD model (pre-trained on millions of images) + const cocoSsd = await import("@tensorflow-models/coco-ssd"); + model = await cocoSsd.load({ + base: "lite_mobilenet_v2", // Smaller, faster model + }); +} + +async function detectPanelsML(imageData: ImageData): Promise { + await loadModel(); + + // Create HTMLCanvasElement to run model inference + const canvas = document.createElement("canvas"); + canvas.width = imageData.width; + canvas.height = imageData.height; + const ctx = canvas.getContext("2d")!; + ctx.putImageData(imageData, 0, 0); + + // Run COCO-SSD model + const predictions = await model.detect(canvas); + + // Filter predictions to find rectangular regions (panels) + // COCO-SSD detects common objects, we look for rectangular ones + const panels: Panel[] = []; + const imgWidth = imageData.width; + const imgHeight = imageData.height; + + for (let i = 0; i < predictions.length; i++) { + const pred = predictions[i]; + + // COCO-SSD detects "book" and similar objects + // We filter for reasonable panel-like detections + const [x, y, w, h] = pred.bbox; + const aspectRatio = w / h; + + const isRectangular = + aspectRatio > 0.3 && // Not too tall/thin + aspectRatio < 5 && // Not too wide + w > imgWidth * 0.05 && // Not too small + h > imgHeight * 0.05; + + if (isRectangular) { + panels.push({ + id: `ml-panel-${i}`, + x: (x / imgWidth) * 100, + y: (y / imgHeight) * 100, + width: (w / imgWidth) * 100, + height: (h / imgHeight) * 100, + reading_order: i, + }); + } + } + + // Sort panels by reading order + panels.sort((a, b) => { + const rowA = Math.floor(a.y / 25); + const rowB = Math.floor(b.y / 25); + if (rowA !== rowB) return rowA - rowB; + return a.x - b.x; + }); + + panels.forEach((p, i) => (p.reading_order = i)); + + return panels; +} + +export { detectPanelsML, loadModel }; +``` + +--- + +### 4. Grid Detection (`panel-detector.ts` - Update) + +Modify the existing `panel-detector.ts` to add the export at the end: + +```typescript +// Grid-based panel detection (fast, lightweight) +// Keep as final fallback + +interface Panel { + id: string; + x: number; + y: number; + width: number; + height: number; + reading_order: number; +} + +interface GridConfig { + rows: number; + cols: number; +} + +function detectPanelsGrid( + imageData: ImageData, + config: GridConfig = { rows: 3, cols: 3 }, +): Panel[] { + const panels: Panel[] = []; + const cellWidth = imageData.width / config.cols; + const cellHeight = imageData.height / config.rows; + + for (let y = 0; y < config.rows; y++) { + for (let x = 0; x < config.cols; x++) { + const cell = extractCell(imageData, x, y, cellWidth, cellHeight); + + if (!isEmpty(cell)) { + panels.push({ + id: `panel-${panels.length}`, + x: (x / config.cols) * 100, + y: (y / config.rows) * 100, + width: (1 / config.cols) * 100, + height: (1 / config.rows) * 100, + reading_order: panels.length, + }); + } + } + } + + return mergeAdjacentPanels(panels); +} + +function isEmpty(cellData: ImageData): boolean { + let emptyPixels = 0; + const totalPixels = cellData.width * cellData.height; + const threshold = 0.95; + + for (let i = 0; i < cellData.data.length; i += 4) { + const r = cellData.data[i]; + const g = cellData.data[i + 1]; + const b = cellData.data[i + 2]; + const a = cellData.data[i + 3]; + + if (a < 10 || (r > 250 && g > 250 && b > 250)) { + emptyPixels++; + } + } + + return emptyPixels / totalPixels > threshold; +} + +function mergeAdjacentPanels(panels: Panel[]): Panel[] { + const merged: Panel[] = []; + const used = new Set(); + + for (let i = 0; i < panels.length; i++) { + if (used.has(i)) continue; + + let current = { ...panels[i] }; + used.add(i); + + for (let j = i + 1; j < panels.length; j++) { + if (used.has(j)) continue; + if (isAdjacent(current, panels[j])) { + current = mergePanels(current, panels[j]); + used.add(j); + } + } + + merged.push(current); + } + + return merged; +} + +function extractCell( + imageData: ImageData, + gridX: number, + gridY: number, + cellWidth: number, + cellHeight: number, +): ImageData { + const startX = Math.floor(gridX * cellWidth); + const startY = Math.floor(gridY * cellHeight); + const width = Math.floor(cellWidth); + const height = Math.floor(cellHeight); + + const cellData = new Uint8ClampedArray(width * height * 4); + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + const srcIdx = ((startY + y) * imageData.width + (startX + x)) * 4; + const destIdx = (y * width + x) * 4; + cellData[destIdx] = imageData.data[srcIdx]; + cellData[destIdx + 1] = imageData.data[srcIdx + 1]; + cellData[destIdx + 2] = imageData.data[srcIdx + 2]; + cellData[destIdx + 3] = imageData.data[srcIdx + 3]; + } + } + + return new ImageData(cellData, width, height); +} + +function isAdjacent(p1: Panel, p2: Panel): boolean { + const tolerance = 5; + if (Math.abs(p1.y - p2.y) < tolerance && Math.abs(p1.height - p2.height) < tolerance) { + return Math.abs(p1.x + p1.width - p2.x) < tolerance || Math.abs(p2.x + p2.width - p1.x) < tolerance; + } + if (Math.abs(p1.x - p2.x) < tolerance && Math.abs(p1.width - p2.width) < tolerance) { + return Math.abs(p1.y + p1.height - p2.y) < tolerance || Math.abs(p2.y + p2.height - p1.y) < tolerance; + } + return false; +} + +function mergePanels(p1: Panel, p2: Panel): Panel { + const minX = Math.min(p1.x, p2.x); + const minY = Math.min(p1.y, p2.y); + const maxX = Math.max(p1.x + p1.width, p2.x + p2.width); + const maxY = Math.max(p1.y + p1.height, p2.y + p2.height); + + return { + id: p1.id, + x: minX, + y: minY, + width: maxX - minX, + height: maxY - minY, + reading_order: Math.min(p1.reading_order, p2.reading_order), + }; +} + +// ADD THIS EXPORT AT THE END OF THE FILE +export { detectPanelsGrid, isEmpty, mergeAdjacentPanels, extractCell, isAdjacent, mergePanels }; +``` + +--- + +### 5. Panel Editor Updates (`panel-editor.ts`) + +Modify the existing `panel-editor.ts` to add imports and re-detect function: + +```typescript +// Manual panel editor for admins/power users + +import { Alpine } from "../../alpine"; +import { apiPut } from "../../api"; +import { detectPanels, Panel } from "./panel-detection.service"; + +async function loadImageForPage(pageNumber: number): Promise { + const mediaItemId = document.body.dataset.mediaItemId; + if (!mediaItemId) { + throw new Error("No mediaItemId found"); + } + + const token = localStorage.getItem("token"); + const response = await fetch(`/readers/${mediaItemId}/pages/${pageNumber}`, { + headers: { Authorization: `Bearer ${token}` }, + }); + + if (!response.ok) { + throw new Error(`Failed to load page ${pageNumber}`); + } + + const blob = await response.blob(); + const img = new Image(); + img.src = URL.createObjectURL(blob); + + await new Promise((resolve) => { + img.onload = () => resolve(); + }); + + return img; +} + +function getCurrentPageNumber(): number { + const Alpine = (window as any).Alpine; + if (Alpine) { + const readerEl = document.querySelector('[x-data="readerShell"]'); + if (readerEl) { + const readerShell = Alpine.$data(readerEl); + if (readerShell?.currentPage) { + return readerShell.currentPage; + } + } + } + + const content = document.getElementById("reader-content"); + const pageFromDataset = content?.dataset.currentPage; + if (pageFromDataset) { + return parseInt(pageFromDataset, 10); + } + + return 1; +} + +function loadPage(pageNumber: number): void { + window.dispatchEvent( + new CustomEvent("navigate-to-page", { detail: { page: pageNumber } }), + ); +} + +function openPanelEditor(pageNumber: number): void { + const modal = document.getElementById("panel-editor-modal"); + modal?.classList.remove("hidden"); + + const canvas = document.getElementById("panel-editor-canvas") as HTMLCanvasElement; + const ctx = canvas?.getContext("2d"); + + loadImageForPage(pageNumber).then((image) => { + canvas!.width = image.width; + canvas!.height = image.height; + ctx?.drawImage(image, 0, 0); + + enablePanelDrawing(canvas!); + }); +} + +function enablePanelDrawing(canvas: HTMLCanvasElement): void { + let isDrawing = false; + let startX = 0; + let startY = 0; + + canvas.addEventListener("mousedown", (e) => { + isDrawing = true; + startX = e.offsetX; + startY = e.offsetY; + }); + + canvas.addEventListener("mousemove", (e) => { + if (!isDrawing) return; + + const ctx = canvas.getContext("2d"); + // Clear and redraw to show selection rectangle + ctx?.clearRect(0, 0, canvas.width, canvas.height); + ctx?.drawImage(canvas, 0, 0); + ctx?.strokeRect(startX, startY, e.offsetX - startX, e.offsetY - startY); + }); + + canvas.addEventListener("mouseup", (e) => { + if (!isDrawing) return; + isDrawing = false; + + const panel: Panel = { + id: `manual-${Date.now()}`, + x: (startX / canvas.width) * 100, + y: (startY / canvas.height) * 100, + width: ((e.offsetX - startX) / canvas.width) * 100, + height: ((e.offsetY - startY) / canvas.height) * 100, + reading_order: 0, + }; + + saveManualPanel(panel); + }); +} + +async function saveManualPanel(panel: Panel): Promise { + const mediaItemId = document.body.dataset.mediaItemId; + const pageNumber = getCurrentPageNumber(); + + await apiPut(`/readers/${mediaItemId}/panels/${pageNumber}`, { + detection_method: "manual", + panels: [panel], + }); + + loadPage(pageNumber); +} + +// Re-detect panels using detection service +async function reDetectPanels(pageNumber: number): Promise { + const image = await loadImageForPage(pageNumber); + + const canvas = document.createElement("canvas"); + canvas.width = image.width; + canvas.height = image.height; + const ctx = canvas.getContext("2d")!; + ctx.drawImage(image, 0, 0); + + const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height); + const result = await detectPanels(imageData, true); + + return result.panels; +} + +// Alpine component +Alpine.data("panelEditor", () => ({ + get isComicOrManga(): boolean { + const libraryType = document.body.dataset.mediaType; + return libraryType === "comic" || libraryType === "manga"; + }, + + openPanelEditor(pageNumber: number) { + openPanelEditor(pageNumber); + }, + + async reDetectPanels(pageNumber: number) { + const panels = await reDetectPanels(pageNumber); + return panels; + } +})); + +export { openPanelEditor, reDetectPanels }; +``` + +--- + +### 6. Page Cache Integration (`page-cache.ts` - Optional) + +Optional: Add on-demand panel detection to page-cache.ts: + +```typescript +// Add this import at the top +import { detectPanels } from "./panel-detection.service"; + +// Add to PageCacheState interface +interface PageCacheState { + cache: Map; + loading: Set; + maxAhead: number; + mediaItemId: string; + panelData: Map; +} + +// Add this function +async function detectPagePanels( + state: PageCacheState, + pageNumber: number +): Promise { + // Check if already detected + if (state.panelData?.has(pageNumber)) { + return state.panelData.get(pageNumber)!.panels; + } + + // Get or create image + let image: HTMLImageElement; + if (state.cache.has(pageNumber)) { + image = state.cache.get(pageNumber)!; + } else { + image = await loadComicPage(state, pageNumber); + } + + // Run detection on demand + const canvas = document.createElement("canvas"); + canvas.width = image.width; + canvas.height = image.height; + const ctx = canvas.getContext("2d")!; + ctx.drawImage(image, 0, 0); + const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height); + + const result = await detectPanels(imageData, true); + + if (!state.panelData) { + state.panelData = new Map(); + } + state.panelData.set(pageNumber, result); + + return result.panels; +} + +// Export the new function +export { createPageCache, getCachedPage, loadComicPage, detectPagePanels }; +``` + +--- + +## Implementation Order + +1. **Add dependencies to `package.json`** and run `npm install` +2. **Create `panel-detection.service.ts`** +3. **Create `panel-detection.opencv.ts`** +4. **Create `panel-detection.ml.ts`** +5. **Update `panel-detector.ts`** - add export statement (one line at the end) +6. **Update `panel-editor.ts`** - add imports and re-detect function +7. **(Optional) Update `page-cache.ts`** - add on-demand detection + +--- + +## Future Enhancements + +1. **User feedback loop:** Store user corrections to improve detection +2. **Per-comic detection:** Different methods for different comic styles +3. **Batch detection:** Pre-detect pages in background +4. **Detection history:** Track which method works best per comic +5. **Panel preview:** Show detected panels before entering panel view