Add panel detection implementation plan documentation

- Document multi-tier detection pipeline: OpenCV → ML (COCO-SSD) → Grid → Manual Editor - Include dependency details with bundle sizes (OpenCV ~500KB, TensorFlow.js ~2MB) - Add architecture overview and fallback chain explanation
2026-04-04 01:01:14 -04:00
parent 0c199963b9
commit 1d4195ba5e
1 changed files with 777 additions and 0 deletions
@@ -0,0 +1,777 @@
+# Panel Detection Implementation Plan
+
+## Overview
+
+Multi-tier panel detection system with fallback chain:
+**OpenCV → ML (COCO-SSD) → Grid → Manual Editor**
+
+Designed for a constantly growing library - handles any comic style without custom training.
+
+---
+
+## Detection Pipeline
+
+```
+1. OpenCV Edge Detection (Primary)
+   ├─ Fast, lightweight (~500KB lazy-loaded)
+   ├─ Works on 80% of comics with clear panel borders
+   └─ Future-proof: works on unknown future comics
+
+2. ML Detection (COCO-SSD Fallback)
+   ├─ Pre-trained on millions of diverse images
+   ├─ Handles irregular layouts
+   └─ ~2MB (TensorFlow.js) + ~2MB (model), lazy-loaded
+
+3. Grid Detection (Baseline)
+   └─ Always works as final fallback
+
+4. Manual Editor (Last Resort)
+   └─ User manually draws panels
+```
+
+---
+
+## Dependencies
+
+Add to `package.json`:
+
+```json
+{
+  "dependencies": {
+    "@techstark/opencv-js": "^4.12.0",
+    "@tensorflow/tfjs": "^4.22.0",
+    "@tensorflow-models/coco-ssd": "^2.2.3"
+  }
+}
+```
+
+**Bundle sizes:**
+- OpenCV.js: ~500KB (lazy-loaded)
+- TensorFlow.js: ~2MB (lazy-loaded)
+- COCO-SSD model: ~2MB (lazy-loaded, cached after first load)
+- **Total: ~4.5MB** (acceptable for modern networks)
+
+---
+
+## File Structure
+
+```
+web/src/reader/comic/
+├── panel-detection.service.ts    [NEW] - Main detection service with fallback chain
+├── panel-detection.opencv.ts      [NEW] - OpenCV edge detection
+├── panel-detection.ml.ts         [NEW] - COCO-SSD ML detection
+├── panel-detector.ts              [MODIFY] - Add export for grid detection
+├── panel-editor.ts               [MODIFY] - Add re-detect, connect to service
+├── page-cache.ts                  [OPTIONAL] - On-demand detection
+├── background-color.ts           [KEEP]
+├── chapter-markers.ts             [KEEP]
+├── page-order.ts                 [KEEP]
+├── page-scrubber.ts              [KEEP]
+└── panel-gap.ts                   [KEEP]
+```
+
+---
+
+## Implementation
+
+### 1. Panel Detection Service (`panel-detection.service.ts`)
+
+Create this file in `web/src/reader/comic/`:
+
+```typescript
+// Main panel detection service with fallback chain
+// Priority: OpenCV → ML → Grid → Manual Editor
+
+interface DetectionResult {
+  panels: Panel[];
+  method: "opencv" | "ml" | "grid" | "manual";
+  confidence: number;
+}
+
+interface Panel {
+  id: string;
+  x: number;
+  y: number;
+  width: number;
+  height: number;
+  reading_order: number;
+}
+
+async function detectPanels(
+  imageData: ImageData,
+  allowManual: boolean = true
+): Promise<DetectionResult> {
+  
+  // Tier 1: OpenCV Edge Detection
+  try {
+    const panels = await detectPanelsOpenCV(imageData);
+    if (validatePanels(panels, imageData)) {
+      return { panels, method: "opencv", confidence: 0.85 };
+    }
+  } catch (e) {
+    console.warn("OpenCV detection failed:", e);
+  }
+  
+  // Tier 2: ML Detection (COCO-SSD)
+  try {
+    const panels = await detectPanelsML(imageData);
+    if (validatePanels(panels, imageData)) {
+      return { panels, method: "ml", confidence: 0.9 };
+    }
+  } catch (e) {
+    console.warn("ML detection failed:", e);
+  }
+  
+  // Tier 3: Grid Detection (baseline)
+  const panels = detectPanelsGrid(imageData);
+  return { panels, method: "grid", confidence: 0.5 };
+}
+
+function validatePanels(panels: Panel[], imageData: ImageData): boolean {
+  // Must have at least 1 panel
+  if (panels.length === 0) return false;
+  
+  // Should not have too many panels (probably noise)
+  if (panels.length > 30) return false;
+  
+  // Panels should cover reasonable area (not all empty space)
+  let totalArea = panels.reduce((sum, p) => sum + (p.width * p.height), 0);
+  if (totalArea < 10 || totalArea > 100) return false;
+  
+  return true;
+}
+
+// Import detection methods from other files
+async function detectPanelsOpenCV(imageData: ImageData): Promise<Panel[]>;
+async function detectPanelsML(imageData: ImageData): Promise<Panel[]>;
+function detectPanelsGrid(imageData: ImageData, config?: { rows: number; cols: number }): Panel[];
+
+export { detectPanels, DetectionResult, Panel };
+```
+
+---
+
+### 2. OpenCV Detection (`panel-detection.opencv.ts`)
+
+Create this file in `web/src/reader/comic/`:
+
+```typescript
+// OpenCV.js-based edge detection for panel boundaries
+
+interface Panel {
+  id: string;
+  x: number;
+  y: number;
+  width: number;
+  height: number;
+  reading_order: number;
+}
+
+let openCVLoaded = false;
+
+async function loadOpenCV(): Promise<void> {
+  if (openCVLoaded) return;
+  
+  // OpenCV.js loads asynchronously and registers globally
+  await import("@techstark/opencv-js");
+  
+  // Wait for OpenCV to be ready
+  return new Promise<void>((resolve) => {
+    const check = () => {
+      if ((window as any).cv && (window as any).cv.Mat) {
+        openCVLoaded = true;
+        resolve();
+      } else {
+        setTimeout(check, 50);
+      }
+    };
+    check();
+  });
+}
+
+async function detectPanelsOpenCV(imageData: ImageData): Promise<Panel[]> {
+  await loadOpenCV();
+  
+  const cv = (window as any).cv;
+  
+  // Create matrices from ImageData
+  const src = cv.matFromImageData(imageData);
+  const gray = new cv.Mat();
+  const blurred = new cv.Mat();
+  const edges = new cv.Mat();
+  const contours = new cv.Mat();
+  const hierarchy = new cv.Mat();
+  
+  try {
+    // Convert to grayscale
+    cv.cvtColor(src, gray, cv.COLOR_RGBA2GRAY, 0);
+    
+    // Apply Gaussian blur to reduce noise
+    cv.GaussianBlur(gray, blurred, new cv.Size(5, 5), 0, 0, cv.BORDER_DEFAULT);
+    
+    // Detect edges using Canny
+    cv.Canny(blurred, edges, 50, 150, 3, false);
+    
+    // Find contours
+    cv.findContours(
+      edges, 
+      contours, 
+      hierarchy, 
+      cv.RETR_EXTERNAL, 
+      cv.CHAIN_APPROX_SIMPLE
+    );
+    
+    // Convert contours to panels
+    const panels: Panel[] = [];
+    const imgWidth = imageData.width;
+    const imgHeight = imageData.height;
+    
+    for (let i = 0; i < contours.size(); i++) {
+      const rect = cv.boundingRect(contours.get(i));
+      const aspectRatio = rect.width / rect.height;
+      
+      // Filter: reject very small or very thin contours
+      const minSize = Math.min(imgWidth, imgHeight) * 0.05;
+      if (rect.width < minSize || rect.height < minSize) continue;
+      if (aspectRatio < 0.1 || aspectRatio > 10) continue;
+      
+      panels.push({
+        id: `opencv-panel-${i}`,
+        x: (rect.x / imgWidth) * 100,
+        y: (rect.y / imgHeight) * 100,
+        width: (rect.width / imgWidth) * 100,
+        height: (rect.height / imgHeight) * 100,
+        reading_order: i,
+      });
+    }
+    
+    // Sort panels by reading order (top-left to bottom-right)
+    panels.sort((a, b) => {
+      const rowA = Math.floor(a.y / 25);
+      const rowB = Math.floor(b.y / 25);
+      if (rowA !== rowB) return rowA - rowB;
+      return a.x - b.x;
+    });
+    
+    // Reassign reading order after sorting
+    panels.forEach((p, i) => (p.reading_order = i));
+    
+    return panels;
+    
+  } finally {
+    // Clean up OpenCV matrices
+    src.delete();
+    gray.delete();
+    blurred.delete();
+    edges.delete();
+    contours.delete();
+    hierarchy.delete();
+  }
+}
+
+export { detectPanelsOpenCV, loadOpenCV };
+```
+
+---
+
+### 3. ML Detection (`panel-detection.ml.ts`)
+
+Create this file in `web/src/reader/comic/`:
+
+```typescript
+// ML-based panel detection using COCO-SSD pre-trained model
+
+interface Panel {
+  id: string;
+  x: number;
+  y: number;
+  width: number;
+  height: number;
+  reading_order: number;
+}
+
+let model: any = null;
+let tfLoaded = false;
+
+async function loadTF(): Promise<void> {
+  if (tfLoaded) return;
+  
+  // Load TensorFlow.js
+  await import("@tensorflow/tfjs");
+  tfLoaded = true;
+}
+
+async function loadModel(): Promise<void> {
+  if (model) return;
+  
+  await loadTF();
+  
+  // Load COCO-SSD model (pre-trained on millions of images)
+  const cocoSsd = await import("@tensorflow-models/coco-ssd");
+  model = await cocoSsd.load({
+    base: "lite_mobilenet_v2", // Smaller, faster model
+  });
+}
+
+async function detectPanelsML(imageData: ImageData): Promise<Panel[]> {
+  await loadModel();
+  
+  // Create HTMLCanvasElement to run model inference
+  const canvas = document.createElement("canvas");
+  canvas.width = imageData.width;
+  canvas.height = imageData.height;
+  const ctx = canvas.getContext("2d")!;
+  ctx.putImageData(imageData, 0, 0);
+  
+  // Run COCO-SSD model
+  const predictions = await model.detect(canvas);
+  
+  // Filter predictions to find rectangular regions (panels)
+  // COCO-SSD detects common objects, we look for rectangular ones
+  const panels: Panel[] = [];
+  const imgWidth = imageData.width;
+  const imgHeight = imageData.height;
+  
+  for (let i = 0; i < predictions.length; i++) {
+    const pred = predictions[i];
+    
+    // COCO-SSD detects "book" and similar objects
+    // We filter for reasonable panel-like detections
+    const [x, y, w, h] = pred.bbox;
+    const aspectRatio = w / h;
+    
+    const isRectangular = 
+      aspectRatio > 0.3 &&  // Not too tall/thin
+      aspectRatio < 5 &&     // Not too wide
+      w > imgWidth * 0.05 && // Not too small
+      h > imgHeight * 0.05;
+    
+    if (isRectangular) {
+      panels.push({
+        id: `ml-panel-${i}`,
+        x: (x / imgWidth) * 100,
+        y: (y / imgHeight) * 100,
+        width: (w / imgWidth) * 100,
+        height: (h / imgHeight) * 100,
+        reading_order: i,
+      });
+    }
+  }
+  
+  // Sort panels by reading order
+  panels.sort((a, b) => {
+    const rowA = Math.floor(a.y / 25);
+    const rowB = Math.floor(b.y / 25);
+    if (rowA !== rowB) return rowA - rowB;
+    return a.x - b.x;
+  });
+  
+  panels.forEach((p, i) => (p.reading_order = i));
+  
+  return panels;
+}
+
+export { detectPanelsML, loadModel };
+```
+
+---
+
+### 4. Grid Detection (`panel-detector.ts` - Update)
+
+Modify the existing `panel-detector.ts` to add the export at the end:
+
+```typescript
+// Grid-based panel detection (fast, lightweight)
+// Keep as final fallback
+
+interface Panel {
+  id: string;
+  x: number;
+  y: number;
+  width: number;
+  height: number;
+  reading_order: number;
+}
+
+interface GridConfig {
+  rows: number;
+  cols: number;
+}
+
+function detectPanelsGrid(
+  imageData: ImageData,
+  config: GridConfig = { rows: 3, cols: 3 },
+): Panel[] {
+  const panels: Panel[] = [];
+  const cellWidth = imageData.width / config.cols;
+  const cellHeight = imageData.height / config.rows;
+
+  for (let y = 0; y < config.rows; y++) {
+    for (let x = 0; x < config.cols; x++) {
+      const cell = extractCell(imageData, x, y, cellWidth, cellHeight);
+
+      if (!isEmpty(cell)) {
+        panels.push({
+          id: `panel-${panels.length}`,
+          x: (x / config.cols) * 100,
+          y: (y / config.rows) * 100,
+          width: (1 / config.cols) * 100,
+          height: (1 / config.rows) * 100,
+          reading_order: panels.length,
+        });
+      }
+    }
+  }
+
+  return mergeAdjacentPanels(panels);
+}
+
+function isEmpty(cellData: ImageData): boolean {
+  let emptyPixels = 0;
+  const totalPixels = cellData.width * cellData.height;
+  const threshold = 0.95;
+
+  for (let i = 0; i < cellData.data.length; i += 4) {
+    const r = cellData.data[i];
+    const g = cellData.data[i + 1];
+    const b = cellData.data[i + 2];
+    const a = cellData.data[i + 3];
+
+    if (a < 10 || (r > 250 && g > 250 && b > 250)) {
+      emptyPixels++;
+    }
+  }
+
+  return emptyPixels / totalPixels > threshold;
+}
+
+function mergeAdjacentPanels(panels: Panel[]): Panel[] {
+  const merged: Panel[] = [];
+  const used = new Set<number>();
+
+  for (let i = 0; i < panels.length; i++) {
+    if (used.has(i)) continue;
+
+    let current = { ...panels[i] };
+    used.add(i);
+
+    for (let j = i + 1; j < panels.length; j++) {
+      if (used.has(j)) continue;
+      if (isAdjacent(current, panels[j])) {
+        current = mergePanels(current, panels[j]);
+        used.add(j);
+      }
+    }
+
+    merged.push(current);
+  }
+
+  return merged;
+}
+
+function extractCell(
+  imageData: ImageData,
+  gridX: number,
+  gridY: number,
+  cellWidth: number,
+  cellHeight: number,
+): ImageData {
+  const startX = Math.floor(gridX * cellWidth);
+  const startY = Math.floor(gridY * cellHeight);
+  const width = Math.floor(cellWidth);
+  const height = Math.floor(cellHeight);
+
+  const cellData = new Uint8ClampedArray(width * height * 4);
+  for (let y = 0; y < height; y++) {
+    for (let x = 0; x < width; x++) {
+      const srcIdx = ((startY + y) * imageData.width + (startX + x)) * 4;
+      const destIdx = (y * width + x) * 4;
+      cellData[destIdx] = imageData.data[srcIdx];
+      cellData[destIdx + 1] = imageData.data[srcIdx + 1];
+      cellData[destIdx + 2] = imageData.data[srcIdx + 2];
+      cellData[destIdx + 3] = imageData.data[srcIdx + 3];
+    }
+  }
+
+  return new ImageData(cellData, width, height);
+}
+
+function isAdjacent(p1: Panel, p2: Panel): boolean {
+  const tolerance = 5;
+  if (Math.abs(p1.y - p2.y) < tolerance && Math.abs(p1.height - p2.height) < tolerance) {
+    return Math.abs(p1.x + p1.width - p2.x) < tolerance || Math.abs(p2.x + p2.width - p1.x) < tolerance;
+  }
+  if (Math.abs(p1.x - p2.x) < tolerance && Math.abs(p1.width - p2.width) < tolerance) {
+    return Math.abs(p1.y + p1.height - p2.y) < tolerance || Math.abs(p2.y + p2.height - p1.y) < tolerance;
+  }
+  return false;
+}
+
+function mergePanels(p1: Panel, p2: Panel): Panel {
+  const minX = Math.min(p1.x, p2.x);
+  const minY = Math.min(p1.y, p2.y);
+  const maxX = Math.max(p1.x + p1.width, p2.x + p2.width);
+  const maxY = Math.max(p1.y + p1.height, p2.y + p2.height);
+
+  return {
+    id: p1.id,
+    x: minX,
+    y: minY,
+    width: maxX - minX,
+    height: maxY - minY,
+    reading_order: Math.min(p1.reading_order, p2.reading_order),
+  };
+}
+
+// ADD THIS EXPORT AT THE END OF THE FILE
+export { detectPanelsGrid, isEmpty, mergeAdjacentPanels, extractCell, isAdjacent, mergePanels };
+```
+
+---
+
+### 5. Panel Editor Updates (`panel-editor.ts`)
+
+Modify the existing `panel-editor.ts` to add imports and re-detect function:
+
+```typescript
+// Manual panel editor for admins/power users
+
+import { Alpine } from "../../alpine";
+import { apiPut } from "../../api";
+import { detectPanels, Panel } from "./panel-detection.service";
+
+async function loadImageForPage(pageNumber: number): Promise<HTMLImageElement> {
+  const mediaItemId = document.body.dataset.mediaItemId;
+  if (!mediaItemId) {
+    throw new Error("No mediaItemId found");
+  }
+
+  const token = localStorage.getItem("token");
+  const response = await fetch(`/readers/${mediaItemId}/pages/${pageNumber}`, {
+    headers: { Authorization: `Bearer ${token}` },
+  });
+
+  if (!response.ok) {
+    throw new Error(`Failed to load page ${pageNumber}`);
+  }
+
+  const blob = await response.blob();
+  const img = new Image();
+  img.src = URL.createObjectURL(blob);
+
+  await new Promise<void>((resolve) => {
+    img.onload = () => resolve();
+  });
+
+  return img;
+}
+
+function getCurrentPageNumber(): number {
+  const Alpine = (window as any).Alpine;
+  if (Alpine) {
+    const readerEl = document.querySelector('[x-data="readerShell"]');
+    if (readerEl) {
+      const readerShell = Alpine.$data(readerEl);
+      if (readerShell?.currentPage) {
+        return readerShell.currentPage;
+      }
+    }
+  }
+
+  const content = document.getElementById("reader-content");
+  const pageFromDataset = content?.dataset.currentPage;
+  if (pageFromDataset) {
+    return parseInt(pageFromDataset, 10);
+  }
+
+  return 1;
+}
+
+function loadPage(pageNumber: number): void {
+  window.dispatchEvent(
+    new CustomEvent("navigate-to-page", { detail: { page: pageNumber } }),
+  );
+}
+
+function openPanelEditor(pageNumber: number): void {
+  const modal = document.getElementById("panel-editor-modal");
+  modal?.classList.remove("hidden");
+
+  const canvas = document.getElementById("panel-editor-canvas") as HTMLCanvasElement;
+  const ctx = canvas?.getContext("2d");
+
+  loadImageForPage(pageNumber).then((image) => {
+    canvas!.width = image.width;
+    canvas!.height = image.height;
+    ctx?.drawImage(image, 0, 0);
+
+    enablePanelDrawing(canvas!);
+  });
+}
+
+function enablePanelDrawing(canvas: HTMLCanvasElement): void {
+  let isDrawing = false;
+  let startX = 0;
+  let startY = 0;
+
+  canvas.addEventListener("mousedown", (e) => {
+    isDrawing = true;
+    startX = e.offsetX;
+    startY = e.offsetY;
+  });
+
+  canvas.addEventListener("mousemove", (e) => {
+    if (!isDrawing) return;
+
+    const ctx = canvas.getContext("2d");
+    // Clear and redraw to show selection rectangle
+    ctx?.clearRect(0, 0, canvas.width, canvas.height);
+    ctx?.drawImage(canvas, 0, 0);
+    ctx?.strokeRect(startX, startY, e.offsetX - startX, e.offsetY - startY);
+  });
+
+  canvas.addEventListener("mouseup", (e) => {
+    if (!isDrawing) return;
+    isDrawing = false;
+
+    const panel: Panel = {
+      id: `manual-${Date.now()}`,
+      x: (startX / canvas.width) * 100,
+      y: (startY / canvas.height) * 100,
+      width: ((e.offsetX - startX) / canvas.width) * 100,
+      height: ((e.offsetY - startY) / canvas.height) * 100,
+      reading_order: 0,
+    };
+
+    saveManualPanel(panel);
+  });
+}
+
+async function saveManualPanel(panel: Panel): Promise<void> {
+  const mediaItemId = document.body.dataset.mediaItemId;
+  const pageNumber = getCurrentPageNumber();
+
+  await apiPut(`/readers/${mediaItemId}/panels/${pageNumber}`, {
+    detection_method: "manual",
+    panels: [panel],
+  });
+
+  loadPage(pageNumber);
+}
+
+// Re-detect panels using detection service
+async function reDetectPanels(pageNumber: number): Promise<Panel[]> {
+  const image = await loadImageForPage(pageNumber);
+  
+  const canvas = document.createElement("canvas");
+  canvas.width = image.width;
+  canvas.height = image.height;
+  const ctx = canvas.getContext("2d")!;
+  ctx.drawImage(image, 0, 0);
+  
+  const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height);
+  const result = await detectPanels(imageData, true);
+  
+  return result.panels;
+}
+
+// Alpine component
+Alpine.data("panelEditor", () => ({
+  get isComicOrManga(): boolean {
+    const libraryType = document.body.dataset.mediaType;
+    return libraryType === "comic" || libraryType === "manga";
+  },
+  
+  openPanelEditor(pageNumber: number) {
+    openPanelEditor(pageNumber);
+  },
+  
+  async reDetectPanels(pageNumber: number) {
+    const panels = await reDetectPanels(pageNumber);
+    return panels;
+  }
+}));
+
+export { openPanelEditor, reDetectPanels };
+```
+
+---
+
+### 6. Page Cache Integration (`page-cache.ts` - Optional)
+
+Optional: Add on-demand panel detection to page-cache.ts:
+
+```typescript
+// Add this import at the top
+import { detectPanels } from "./panel-detection.service";
+
+// Add to PageCacheState interface
+interface PageCacheState {
+  cache: Map<number, HTMLImageElement>;
+  loading: Set<number>;
+  maxAhead: number;
+  mediaItemId: string;
+  panelData: Map<number, { panels: any[]; method: string; confidence: number }>;
+}
+
+// Add this function
+async function detectPagePanels(
+  state: PageCacheState,
+  pageNumber: number
+): Promise<any[]> {
+  // Check if already detected
+  if (state.panelData?.has(pageNumber)) {
+    return state.panelData.get(pageNumber)!.panels;
+  }
+  
+  // Get or create image
+  let image: HTMLImageElement;
+  if (state.cache.has(pageNumber)) {
+    image = state.cache.get(pageNumber)!;
+  } else {
+    image = await loadComicPage(state, pageNumber);
+  }
+  
+  // Run detection on demand
+  const canvas = document.createElement("canvas");
+  canvas.width = image.width;
+  canvas.height = image.height;
+  const ctx = canvas.getContext("2d")!;
+  ctx.drawImage(image, 0, 0);
+  const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height);
+  
+  const result = await detectPanels(imageData, true);
+  
+  if (!state.panelData) {
+    state.panelData = new Map();
+  }
+  state.panelData.set(pageNumber, result);
+  
+  return result.panels;
+}
+
+// Export the new function
+export { createPageCache, getCachedPage, loadComicPage, detectPagePanels };
+```
+
+---
+
+## Implementation Order
+
+1. **Add dependencies to `package.json`** and run `npm install`
+2. **Create `panel-detection.service.ts`**
+3. **Create `panel-detection.opencv.ts`**
+4. **Create `panel-detection.ml.ts`**
+5. **Update `panel-detector.ts`** - add export statement (one line at the end)
+6. **Update `panel-editor.ts`** - add imports and re-detect function
+7. **(Optional) Update `page-cache.ts`** - add on-demand detection
+
+---
+
+## Future Enhancements
+
+1. **User feedback loop:** Store user corrections to improve detection
+2. **Per-comic detection:** Different methods for different comic styles
+3. **Batch detection:** Pre-detect pages in background
+4. **Detection history:** Track which method works best per comic
+5. **Panel preview:** Show detected panels before entering panel view