bookhoard/web/src/reader/formats/comic/panel-detection.opencv.ts

// OpenCV.js-based edge detection for panel boundaries

interface Panel {
  id: string;
  x: number;
  y: number;
  width: number;
  height: number;
  reading_order: number;
}

let openCVLoaded = false;

async function loadOpenCV(): Promise<void> {
  if (openCVLoaded) return;

  // OpenCV.js loads asynchronously and registers globally
  await import("@techstark/opencv-js");

  // Wait for OpenCV to be ready
  return new Promise<void>((resolve) => {
    const check = () => {
      if ((window as any).cv && (window as any).cv.Mat) {
        openCVLoaded = true;
        resolve();
      } else {
        setTimeout(check, 50);
      }
    };
    check();
  });
}

async function detectPanelsOpenCV(imageData: ImageData): Promise<Panel[]> {
  await loadOpenCV();

  const cv = (window as any).cv;

  // Create matrices from ImageData
  const src = cv.matFromImageData(imageData);
  const gray = new cv.Mat();
  const blurred = new cv.Mat();
  const edges = new cv.Mat();
  const contours = new cv.Mat();
  const hierarchy = new cv.Mat();

  try {
    // Convert to grayscale
    cv.cvtColor(src, gray, cv.COLOR_RGBA2GRAY, 0);

    // Apply Gaussian blur to reduce noise
    cv.GaussianBlur(gray, blurred, new cv.Size(5, 5), 0, 0, cv.BORDER_DEFAULT);

    // Detect edges using Canny
    cv.Canny(blurred, edges, 50, 150, 3, false);

    // Find contours
    cv.findContours(
      edges,
      contours,
      hierarchy,
      cv.RETR_EXTERNAL,
      cv.CHAIN_APPROX_SIMPLE,
    );

    // Convert contours to panels
    const panels: Panel[] = [];
    const imgWidth = imageData.width;
    const imgHeight = imageData.height;

    for (let i = 0; i < contours.size(); i++) {
      const rect = cv.boundingRect(contours.get(i));
      const aspectRatio = rect.width / rect.height;

      // Filter: reject very small or very thin contours
      const minSize = Math.min(imgWidth, imgHeight) * 0.05;
      if (rect.width < minSize || rect.height < minSize) continue;
      if (aspectRatio < 0.1 || aspectRatio > 10) continue;

      panels.push({
        id: `opencv-panel-${i}`,
        x: (rect.x / imgWidth) * 100,
        y: (rect.y / imgHeight) * 100,
        width: (rect.width / imgWidth) * 100,
        height: (rect.height / imgHeight) * 100,
        reading_order: i,
      });
    }

    // Sort panels by reading order (top-left to bottom-right)
    panels.sort((a, b) => {
      const rowA = Math.floor(a.y / 25);
      const rowB = Math.floor(b.y / 25);
      if (rowA !== rowB) return rowA - rowB;
      return a.x - b.x;
    });

    // Reassign reading order after sorting
    panels.forEach((p, i) => (p.reading_order = i));

    return panels;
  } finally {
    // Clean up OpenCV matrices
    src.delete();
    gray.delete();
    blurred.delete();
    edges.delete();
    contours.delete();
    hierarchy.delete();
  }
}

export { detectPanelsOpenCV, loadOpenCV };