bookhoard/PANEL_DETECTION_PLAN.md

Panel Detection Implementation Plan

Overview

Multi-tier panel detection system with fallback chain: OpenCV → ML (COCO-SSD) → Grid → Manual Editor

Designed for a constantly growing library - handles any comic style without custom training.

---

Detection Pipeline

1. OpenCV Edge Detection (Primary)
   ├─ Fast, lightweight (~500KB lazy-loaded)
   ├─ Works on 80% of comics with clear panel borders
   └─ Future-proof: works on unknown future comics

2. ML Detection (COCO-SSD Fallback)
   ├─ Pre-trained on millions of diverse images
   ├─ Handles irregular layouts
   └─ ~2MB (TensorFlow.js) + ~2MB (model), lazy-loaded

3. Grid Detection (Baseline)
   └─ Always works as final fallback

4. Manual Editor (Last Resort)
   └─ User manually draws panels

---

Dependencies

Add to package.json:

{
  "dependencies": {
    "@techstark/opencv-js": "^4.12.0",
    "@tensorflow/tfjs": "^4.22.0",
    "@tensorflow-models/coco-ssd": "^2.2.3"
  }
}

Bundle sizes:

• OpenCV.js: ~500KB (lazy-loaded)
• TensorFlow.js: ~2MB (lazy-loaded)
• COCO-SSD model: ~2MB (lazy-loaded, cached after first load)
• Total: ~4.5MB (acceptable for modern networks)

---

File Structure

web/src/reader/comic/
├── panel-detection.service.ts    [NEW] - Main detection service with fallback chain
├── panel-detection.opencv.ts      [NEW] - OpenCV edge detection
├── panel-detection.ml.ts         [NEW] - COCO-SSD ML detection
├── panel-detector.ts              [MODIFY] - Add export for grid detection
├── panel-editor.ts               [MODIFY] - Add re-detect, connect to service
├── page-cache.ts                  [OPTIONAL] - On-demand detection
├── background-color.ts           [KEEP]
├── chapter-markers.ts             [KEEP]
├── page-order.ts                 [KEEP]
├── page-scrubber.ts              [KEEP]
└── panel-gap.ts                   [KEEP]

---

Implementation

1. Panel Detection Service (panel-detection.service.ts)

Create this file in web/src/reader/comic/:

// Main panel detection service with fallback chain
// Priority: OpenCV → ML → Grid → Manual Editor

interface DetectionResult {
  panels: Panel[];
  method: "opencv" | "ml" | "grid" | "manual";
  confidence: number;
}

interface Panel {
  id: string;
  x: number;
  y: number;
  width: number;
  height: number;
  reading_order: number;
}

async function detectPanels(
  imageData: ImageData,
  allowManual: boolean = true
): Promise<DetectionResult> {
  
  // Tier 1: OpenCV Edge Detection
  try {
    const panels = await detectPanelsOpenCV(imageData);
    if (validatePanels(panels, imageData)) {
      return { panels, method: "opencv", confidence: 0.85 };
    }
  } catch (e) {
    console.warn("OpenCV detection failed:", e);
  }
  
  // Tier 2: ML Detection (COCO-SSD)
  try {
    const panels = await detectPanelsML(imageData);
    if (validatePanels(panels, imageData)) {
      return { panels, method: "ml", confidence: 0.9 };
    }
  } catch (e) {
    console.warn("ML detection failed:", e);
  }
  
  // Tier 3: Grid Detection (baseline)
  const panels = detectPanelsGrid(imageData);
  return { panels, method: "grid", confidence: 0.5 };
}

function validatePanels(panels: Panel[], imageData: ImageData): boolean {
  // Must have at least 1 panel
  if (panels.length === 0) return false;
  
  // Should not have too many panels (probably noise)
  if (panels.length > 30) return false;
  
  // Panels should cover reasonable area (not all empty space)
  let totalArea = panels.reduce((sum, p) => sum + (p.width * p.height), 0);
  if (totalArea < 10 || totalArea > 100) return false;
  
  return true;
}

// Import detection methods from other files
async function detectPanelsOpenCV(imageData: ImageData): Promise<Panel[]>;
async function detectPanelsML(imageData: ImageData): Promise<Panel[]>;
function detectPanelsGrid(imageData: ImageData, config?: { rows: number; cols: number }): Panel[];

export { detectPanels, DetectionResult, Panel };

---

2. OpenCV Detection (panel-detection.opencv.ts)

Create this file in web/src/reader/comic/:

// OpenCV.js-based edge detection for panel boundaries

interface Panel {
  id: string;
  x: number;
  y: number;
  width: number;
  height: number;
  reading_order: number;
}

let openCVLoaded = false;

async function loadOpenCV(): Promise<void> {
  if (openCVLoaded) return;
  
  // OpenCV.js loads asynchronously and registers globally
  await import("@techstark/opencv-js");
  
  // Wait for OpenCV to be ready
  return new Promise<void>((resolve) => {
    const check = () => {
      if ((window as any).cv && (window as any).cv.Mat) {
        openCVLoaded = true;
        resolve();
      } else {
        setTimeout(check, 50);
      }
    };
    check();
  });
}

async function detectPanelsOpenCV(imageData: ImageData): Promise<Panel[]> {
  await loadOpenCV();
  
  const cv = (window as any).cv;
  
  // Create matrices from ImageData
  const src = cv.matFromImageData(imageData);
  const gray = new cv.Mat();
  const blurred = new cv.Mat();
  const edges = new cv.Mat();
  const contours = new cv.Mat();
  const hierarchy = new cv.Mat();
  
  try {
    // Convert to grayscale
    cv.cvtColor(src, gray, cv.COLOR_RGBA2GRAY, 0);
    
    // Apply Gaussian blur to reduce noise
    cv.GaussianBlur(gray, blurred, new cv.Size(5, 5), 0, 0, cv.BORDER_DEFAULT);
    
    // Detect edges using Canny
    cv.Canny(blurred, edges, 50, 150, 3, false);
    
    // Find contours
    cv.findContours(
      edges, 
      contours, 
      hierarchy, 
      cv.RETR_EXTERNAL, 
      cv.CHAIN_APPROX_SIMPLE
    );
    
    // Convert contours to panels
    const panels: Panel[] = [];
    const imgWidth = imageData.width;
    const imgHeight = imageData.height;
    
    for (let i = 0; i < contours.size(); i++) {
      const rect = cv.boundingRect(contours.get(i));
      const aspectRatio = rect.width / rect.height;
      
      // Filter: reject very small or very thin contours
      const minSize = Math.min(imgWidth, imgHeight) * 0.05;
      if (rect.width < minSize || rect.height < minSize) continue;
      if (aspectRatio < 0.1 || aspectRatio > 10) continue;
      
      panels.push({
        id: `opencv-panel-${i}`,
        x: (rect.x / imgWidth) * 100,
        y: (rect.y / imgHeight) * 100,
        width: (rect.width / imgWidth) * 100,
        height: (rect.height / imgHeight) * 100,
        reading_order: i,
      });
    }
    
    // Sort panels by reading order (top-left to bottom-right)
    panels.sort((a, b) => {
      const rowA = Math.floor(a.y / 25);
      const rowB = Math.floor(b.y / 25);
      if (rowA !== rowB) return rowA - rowB;
      return a.x - b.x;
    });
    
    // Reassign reading order after sorting
    panels.forEach((p, i) => (p.reading_order = i));
    
    return panels;
    
  } finally {
    // Clean up OpenCV matrices
    src.delete();
    gray.delete();
    blurred.delete();
    edges.delete();
    contours.delete();
    hierarchy.delete();
  }
}

export { detectPanelsOpenCV, loadOpenCV };

---

3. ML Detection (panel-detection.ml.ts)

Create this file in web/src/reader/comic/:

// ML-based panel detection using COCO-SSD pre-trained model

interface Panel {
  id: string;
  x: number;
  y: number;
  width: number;
  height: number;
  reading_order: number;
}

let model: any = null;
let tfLoaded = false;

async function loadTF(): Promise<void> {
  if (tfLoaded) return;
  
  // Load TensorFlow.js
  await import("@tensorflow/tfjs");
  tfLoaded = true;
}

async function loadModel(): Promise<void> {
  if (model) return;
  
  await loadTF();
  
  // Load COCO-SSD model (pre-trained on millions of images)
  const cocoSsd = await import("@tensorflow-models/coco-ssd");
  model = await cocoSsd.load({
    base: "lite_mobilenet_v2", // Smaller, faster model
  });
}

async function detectPanelsML(imageData: ImageData): Promise<Panel[]> {
  await loadModel();
  
  // Create HTMLCanvasElement to run model inference
  const canvas = document.createElement("canvas");
  canvas.width = imageData.width;
  canvas.height = imageData.height;
  const ctx = canvas.getContext("2d")!;
  ctx.putImageData(imageData, 0, 0);
  
  // Run COCO-SSD model
  const predictions = await model.detect(canvas);
  
  // Filter predictions to find rectangular regions (panels)
  // COCO-SSD detects common objects, we look for rectangular ones
  const panels: Panel[] = [];
  const imgWidth = imageData.width;
  const imgHeight = imageData.height;
  
  for (let i = 0; i < predictions.length; i++) {
    const pred = predictions[i];
    
    // COCO-SSD detects "book" and similar objects
    // We filter for reasonable panel-like detections
    const [x, y, w, h] = pred.bbox;
    const aspectRatio = w / h;
    
    const isRectangular = 
      aspectRatio > 0.3 &&  // Not too tall/thin
      aspectRatio < 5 &&     // Not too wide
      w > imgWidth * 0.05 && // Not too small
      h > imgHeight * 0.05;
    
    if (isRectangular) {
      panels.push({
        id: `ml-panel-${i}`,
        x: (x / imgWidth) * 100,
        y: (y / imgHeight) * 100,
        width: (w / imgWidth) * 100,
        height: (h / imgHeight) * 100,
        reading_order: i,
      });
    }
  }
  
  // Sort panels by reading order
  panels.sort((a, b) => {
    const rowA = Math.floor(a.y / 25);
    const rowB = Math.floor(b.y / 25);
    if (rowA !== rowB) return rowA - rowB;
    return a.x - b.x;
  });
  
  panels.forEach((p, i) => (p.reading_order = i));
  
  return panels;
}

export { detectPanelsML, loadModel };

---

4. Grid Detection (panel-detector.ts - Update)

Modify the existing panel-detector.ts to add the export at the end:

// Grid-based panel detection (fast, lightweight)
// Keep as final fallback

interface Panel {
  id: string;
  x: number;
  y: number;
  width: number;
  height: number;
  reading_order: number;
}

interface GridConfig {
  rows: number;
  cols: number;
}

function detectPanelsGrid(
  imageData: ImageData,
  config: GridConfig = { rows: 3, cols: 3 },
): Panel[] {
  const panels: Panel[] = [];
  const cellWidth = imageData.width / config.cols;
  const cellHeight = imageData.height / config.rows;

  for (let y = 0; y < config.rows; y++) {
    for (let x = 0; x < config.cols; x++) {
      const cell = extractCell(imageData, x, y, cellWidth, cellHeight);

      if (!isEmpty(cell)) {
        panels.push({
          id: `panel-${panels.length}`,
          x: (x / config.cols) * 100,
          y: (y / config.rows) * 100,
          width: (1 / config.cols) * 100,
          height: (1 / config.rows) * 100,
          reading_order: panels.length,
        });
      }
    }
  }

  return mergeAdjacentPanels(panels);
}

function isEmpty(cellData: ImageData): boolean {
  let emptyPixels = 0;
  const totalPixels = cellData.width * cellData.height;
  const threshold = 0.95;

  for (let i = 0; i < cellData.data.length; i += 4) {
    const r = cellData.data[i];
    const g = cellData.data[i + 1];
    const b = cellData.data[i + 2];
    const a = cellData.data[i + 3];

    if (a < 10 || (r > 250 && g > 250 && b > 250)) {
      emptyPixels++;
    }
  }

  return emptyPixels / totalPixels > threshold;
}

function mergeAdjacentPanels(panels: Panel[]): Panel[] {
  const merged: Panel[] = [];
  const used = new Set<number>();

  for (let i = 0; i < panels.length; i++) {
    if (used.has(i)) continue;

    let current = { ...panels[i] };
    used.add(i);

    for (let j = i + 1; j < panels.length; j++) {
      if (used.has(j)) continue;
      if (isAdjacent(current, panels[j])) {
        current = mergePanels(current, panels[j]);
        used.add(j);
      }
    }

    merged.push(current);
  }

  return merged;
}

function extractCell(
  imageData: ImageData,
  gridX: number,
  gridY: number,
  cellWidth: number,
  cellHeight: number,
): ImageData {
  const startX = Math.floor(gridX * cellWidth);
  const startY = Math.floor(gridY * cellHeight);
  const width = Math.floor(cellWidth);
  const height = Math.floor(cellHeight);

  const cellData = new Uint8ClampedArray(width * height * 4);
  for (let y = 0; y < height; y++) {
    for (let x = 0; x < width; x++) {
      const srcIdx = ((startY + y) * imageData.width + (startX + x)) * 4;
      const destIdx = (y * width + x) * 4;
      cellData[destIdx] = imageData.data[srcIdx];
      cellData[destIdx + 1] = imageData.data[srcIdx + 1];
      cellData[destIdx + 2] = imageData.data[srcIdx + 2];
      cellData[destIdx + 3] = imageData.data[srcIdx + 3];
    }
  }

  return new ImageData(cellData, width, height);
}

function isAdjacent(p1: Panel, p2: Panel): boolean {
  const tolerance = 5;
  if (Math.abs(p1.y - p2.y) < tolerance && Math.abs(p1.height - p2.height) < tolerance) {
    return Math.abs(p1.x + p1.width - p2.x) < tolerance || Math.abs(p2.x + p2.width - p1.x) < tolerance;
  }
  if (Math.abs(p1.x - p2.x) < tolerance && Math.abs(p1.width - p2.width) < tolerance) {
    return Math.abs(p1.y + p1.height - p2.y) < tolerance || Math.abs(p2.y + p2.height - p1.y) < tolerance;
  }
  return false;
}

function mergePanels(p1: Panel, p2: Panel): Panel {
  const minX = Math.min(p1.x, p2.x);
  const minY = Math.min(p1.y, p2.y);
  const maxX = Math.max(p1.x + p1.width, p2.x + p2.width);
  const maxY = Math.max(p1.y + p1.height, p2.y + p2.height);

  return {
    id: p1.id,
    x: minX,
    y: minY,
    width: maxX - minX,
    height: maxY - minY,
    reading_order: Math.min(p1.reading_order, p2.reading_order),
  };
}

// ADD THIS EXPORT AT THE END OF THE FILE
export { detectPanelsGrid, isEmpty, mergeAdjacentPanels, extractCell, isAdjacent, mergePanels };

---

5. Panel Editor Updates (panel-editor.ts)

Modify the existing panel-editor.ts to add imports and re-detect function:

// Manual panel editor for admins/power users

import { Alpine } from "../../alpine";
import { apiPut } from "../../api";
import { detectPanels, Panel } from "./panel-detection.service";

async function loadImageForPage(pageNumber: number): Promise<HTMLImageElement> {
  const mediaItemId = document.body.dataset.mediaItemId;
  if (!mediaItemId) {
    throw new Error("No mediaItemId found");
  }

  const token = localStorage.getItem("token");
  const response = await fetch(`/readers/${mediaItemId}/pages/${pageNumber}`, {
    headers: { Authorization: `Bearer ${token}` },
  });

  if (!response.ok) {
    throw new Error(`Failed to load page ${pageNumber}`);
  }

  const blob = await response.blob();
  const img = new Image();
  img.src = URL.createObjectURL(blob);

  await new Promise<void>((resolve) => {
    img.onload = () => resolve();
  });

  return img;
}

function getCurrentPageNumber(): number {
  const Alpine = (window as any).Alpine;
  if (Alpine) {
    const readerEl = document.querySelector('[x-data="readerShell"]');
    if (readerEl) {
      const readerShell = Alpine.$data(readerEl);
      if (readerShell?.currentPage) {
        return readerShell.currentPage;
      }
    }
  }

  const content = document.getElementById("reader-content");
  const pageFromDataset = content?.dataset.currentPage;
  if (pageFromDataset) {
    return parseInt(pageFromDataset, 10);
  }

  return 1;
}

function loadPage(pageNumber: number): void {
  window.dispatchEvent(
    new CustomEvent("navigate-to-page", { detail: { page: pageNumber } }),
  );
}

function openPanelEditor(pageNumber: number): void {
  const modal = document.getElementById("panel-editor-modal");
  modal?.classList.remove("hidden");

  const canvas = document.getElementById("panel-editor-canvas") as HTMLCanvasElement;
  const ctx = canvas?.getContext("2d");

  loadImageForPage(pageNumber).then((image) => {
    canvas!.width = image.width;
    canvas!.height = image.height;
    ctx?.drawImage(image, 0, 0);

    enablePanelDrawing(canvas!);
  });
}

function enablePanelDrawing(canvas: HTMLCanvasElement): void {
  let isDrawing = false;
  let startX = 0;
  let startY = 0;

  canvas.addEventListener("mousedown", (e) => {
    isDrawing = true;
    startX = e.offsetX;
    startY = e.offsetY;
  });

  canvas.addEventListener("mousemove", (e) => {
    if (!isDrawing) return;

    const ctx = canvas.getContext("2d");
    // Clear and redraw to show selection rectangle
    ctx?.clearRect(0, 0, canvas.width, canvas.height);
    ctx?.drawImage(canvas, 0, 0);
    ctx?.strokeRect(startX, startY, e.offsetX - startX, e.offsetY - startY);
  });

  canvas.addEventListener("mouseup", (e) => {
    if (!isDrawing) return;
    isDrawing = false;

    const panel: Panel = {
      id: `manual-${Date.now()}`,
      x: (startX / canvas.width) * 100,
      y: (startY / canvas.height) * 100,
      width: ((e.offsetX - startX) / canvas.width) * 100,
      height: ((e.offsetY - startY) / canvas.height) * 100,
      reading_order: 0,
    };

    saveManualPanel(panel);
  });
}

async function saveManualPanel(panel: Panel): Promise<void> {
  const mediaItemId = document.body.dataset.mediaItemId;
  const pageNumber = getCurrentPageNumber();

  await apiPut(`/readers/${mediaItemId}/panels/${pageNumber}`, {
    detection_method: "manual",
    panels: [panel],
  });

  loadPage(pageNumber);
}

// Re-detect panels using detection service
async function reDetectPanels(pageNumber: number): Promise<Panel[]> {
  const image = await loadImageForPage(pageNumber);
  
  const canvas = document.createElement("canvas");
  canvas.width = image.width;
  canvas.height = image.height;
  const ctx = canvas.getContext("2d")!;
  ctx.drawImage(image, 0, 0);
  
  const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height);
  const result = await detectPanels(imageData, true);
  
  return result.panels;
}

// Alpine component
Alpine.data("panelEditor", () => ({
  get isComicOrManga(): boolean {
    const libraryType = document.body.dataset.mediaType;
    return libraryType === "comic" || libraryType === "manga";
  },
  
  openPanelEditor(pageNumber: number) {
    openPanelEditor(pageNumber);
  },
  
  async reDetectPanels(pageNumber: number) {
    const panels = await reDetectPanels(pageNumber);
    return panels;
  }
}));

export { openPanelEditor, reDetectPanels };

---

6. Page Cache Integration (page-cache.ts - Optional)

Optional: Add on-demand panel detection to page-cache.ts:

// Add this import at the top
import { detectPanels } from "./panel-detection.service";

// Add to PageCacheState interface
interface PageCacheState {
  cache: Map<number, HTMLImageElement>;
  loading: Set<number>;
  maxAhead: number;
  mediaItemId: string;
  panelData: Map<number, { panels: any[]; method: string; confidence: number }>;
}

// Add this function
async function detectPagePanels(
  state: PageCacheState,
  pageNumber: number
): Promise<any[]> {
  // Check if already detected
  if (state.panelData?.has(pageNumber)) {
    return state.panelData.get(pageNumber)!.panels;
  }
  
  // Get or create image
  let image: HTMLImageElement;
  if (state.cache.has(pageNumber)) {
    image = state.cache.get(pageNumber)!;
  } else {
    image = await loadComicPage(state, pageNumber);
  }
  
  // Run detection on demand
  const canvas = document.createElement("canvas");
  canvas.width = image.width;
  canvas.height = image.height;
  const ctx = canvas.getContext("2d")!;
  ctx.drawImage(image, 0, 0);
  const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height);
  
  const result = await detectPanels(imageData, true);
  
  if (!state.panelData) {
    state.panelData = new Map();
  }
  state.panelData.set(pageNumber, result);
  
  return result.panels;
}

// Export the new function
export { createPageCache, getCachedPage, loadComicPage, detectPagePanels };

---

Implementation Order

1. Add dependencies to package.json and run npm install
2. Create panel-detection.service.ts
3. Create panel-detection.opencv.ts
4. Create panel-detection.ml.ts
5. Update panel-detector.ts - add export statement (one line at the end)
6. Update panel-editor.ts - add imports and re-detect function
7. (Optional) Update page-cache.ts - add on-demand detection

---

Future Enhancements

1. User feedback loop: Store user corrections to improve detection
2. Per-comic detection: Different methods for different comic styles
3. Batch detection: Pre-detect pages in background
4. Detection history: Track which method works best per comic
5. Panel preview: Show detected panels before entering panel view