feat(panel-detection): add metadata extraction and Manga109 TFLite detector

- Add metadata-extractor.js for extracting panel coordinates from Kodansha and Amazon comics - Add manga109-tflite-detector.js for YOLO26-nano model inference - Supports inline CSS linkhotspots and Amazon magnification regions
2026-09-09 11:29:14 -04:00 · 2026-04-16 20:04:03 -04:00
parent e0d53837c6
commit 9d96d81356
2 changed files with 456 additions and 0 deletions
@@ -0,0 +1,203 @@
+// panel-detection/manga109-tflite-detector.js
+// Manga109 panel detection using TFLite
+let tfliteModel = null;
+/**
+ * Initialize the Manga109 TFLite model
+ * @returns {Promise<boolean>} True if loaded successfully
+ */
+export async function initManga109Model() {
+  if (tfliteModel) {
+    console.log("[Manga109 TFLite] Using cached model");
+    return true;
+  }
+  try {
+    console.log("[Manga109 TFLite] Loading model...");
+    // Check if TFLite is available
+    if (typeof tflite === "undefined") {
+      throw new Error(
+        "TensorFlow.js TFLite not loaded. Include @tensorflow/tfjs-tflite",
+      );
+    }
+    // Load the TFLite model directly - NO CONVERSION NEEDED!
+    tfliteModel = await tflite.loadTFLiteModel(
+      "./vendor/manga109/model.tflite",
+    );
+    console.log("[Manga109 TFLite] Model loaded successfully");
+    return true;
+  } catch (error) {
+    console.error("[Manga109 TFLite] Failed to load model:", error);
+    tfliteModel = null;
+    return false;
+  }
+}
+/**
+ * Detect panels using Manga109 TFLite model
+ * @param {ImageData} imageData - Image data from canvas
+ * @returns {Promise<Array>} Array of panel detections
+ */
+export async function detectManga109Panels(imageData) {
+  if (!tfliteModel) {
+    throw new Error("Model not loaded. Call initManga109Model() first.");
+  }
+  console.log("[Manga109 TFLite] Detecting panels...");
+  try {
+    const imgWidth = imageData.width;
+    const imgHeight = imageData.height;
+    // Convert ImageData to tensor
+    const tensor = tf.browser.fromPixels(imageData);
+    // Preprocess: resize to 640x640 (YOLO26 input size)
+    const resized = tf.image.resizeBilinear(tensor, [640, 640]);
+
+    // Normalize to 0-1 and adjust for TFLite model input requirements
+    // YOLO models typically expect RGB in [0, 255] range or [0, 1]
+    // We'll use [0, 1] range
+    const normalized = resized.div(255.0);
+
+    // Add batch dimension: [1, 640, 640, 3]
+    const batched = normalized.expandDims(0);
+    // Run inference
+    const startTime = performance.now();
+    const outputTensor = tfliteModel.predict(batched);
+    const endTime = performance.now();
+    console.log(
+      `[Manga109 TFLite] Inference took ${(endTime - startTime).toFixed(2)}ms`,
+    );
+    // Postprocess outputs
+    const panels = postprocessTFLiteOutputs(outputTensor, imgWidth, imgHeight);
+    console.log(`[Manga109 TFLite] Detected ${panels.length} panels`);
+    // Clean up tensors
+    tensor.dispose();
+    resized.dispose();
+    normalized.dispose();
+    batched.dispose();
+    return panels;
+  } catch (error) {
+    console.error("[Manga109 TFLite] Detection failed:", error);
+    throw error;
+  }
+}
+/**
+ * Postprocess TFLite YOLO outputs to panel detections
+ * @param {Tensor} outputTensor - Model output tensor
+ * @param {number} imgWidth - Original image width
+ * @param {number} imgHeight - Original image height
+ * @returns {Array} Array of panel objects
+ */
+function postprocessTFLiteOutputs(outputTensor, imgWidth, imgHeight) {
+  const panels = [];
+  const confidenceThreshold = 0.5;
+  // TFLite YOLO output format
+  // Typically: [batch, num_detections, 85] for 80 classes (COCO)
+  // For 2-class model (panel, text): [batch, num_detections, 6]
+  // where 6 = [x_center, y_center, width, height, confidence, class_id]
+
+  const outputArray = outputTensor.dataSync();
+  const shape = outputTensor.shape;
+
+  console.log(`[Manga109 TFLite] Output shape: [${shape.join(", ")}]`);
+  const [batch, numDetections, numClasses] = shape;
+  // Parse detections
+  for (let i = 0; i < numDetections; i++) {
+    const offset = i * numClasses;
+    // Extract values (may need adjustment based on actual model output)
+    const centerX = outputArray[offset];
+    const centerY = outputArray[offset + 1];
+    const width = outputArray[offset + 2];
+    const height = outputArray[offset + 3];
+    const confidence = outputArray[offset + 4];
+    const classId = Math.round(outputArray[offset + 5]);
+    // Only detect panels (class 0)
+    if (classId !== 0) continue;
+
+    if (confidence < confidenceThreshold) continue;
+    // Convert from center coords to top-left
+    // YOLO outputs are typically normalized [0, 1] or in pixels
+    // Assuming normalized output:
+    const halfWidth = width / 2;
+    const halfHeight = height / 2;
+    const x = (centerX - halfWidth) * imgWidth;
+    const y = (centerY - halfHeight) * imgHeight;
+    const w = width * imgWidth;
+    const h = height * imgHeight;
+    // Convert to percentages
+    panels.push({
+      id: `manga109-tflite-${panels.length}`,
+      x: (x / imgWidth) * 100,
+      y: (y / imgHeight) * 100,
+      width: (w / imgWidth) * 100,
+      height: (h / imgHeight) * 100,
+      confidence: confidence,
+      reading_order: panels.length,
+    });
+  }
+  // Apply Non-Maximum Suppression
+  return applyNMS(panels);
+}
+
+/**
+ * Apply Non-Maximum Suppression to remove duplicate detections
+ * @param {Array} panels - Array of panel detections
+ * @param {number} iouThreshold - IoU threshold for NMS
+ * @returns {Array} Filtered panels
+ */
+function applyNMS(panels, iouThreshold = 0.5) {
+  if (panels.length === 0) return panels;
+  // Sort by confidence (highest first)
+  panels.sort((a, b) => b.confidence - a.confidence);
+  const keep = [];
+  const suppressed = new Set();
+  for (let i = 0; i < panels.length; i++) {
+    if (suppressed.has(i)) continue;
+    keep.push(panels[i]);
+    // Suppress overlapping boxes
+    for (let j = i + 1; j < panels.length; j++) {
+      if (suppressed.has(j)) continue;
+      const iou = calculateIoU(panels[i], panels[j]);
+      if (iou > iouThreshold) {
+        suppressed.add(j);
+      }
+    }
+  }
+  return keep.map((p, i) => ({ ...p, reading_order: i }));
+}
+/**
+ * Calculate Intersection over Union (IoU)
+ * @param {Object} box1 - First panel
+ * @param {Object} box2 - Second panel
+ * @returns {number} IoU value
+ */
+function calculateIoU(box1, box2) {
+  // Convert percentage coordinates to pixels for calculation
+  const x1 = Math.max(box1.x, box2.x);
+  const y1 = Math.max(box1.y, box2.y);
+  const x2 = Math.min(box1.x + box1.width, box2.x + box2.width);
+  const y2 = Math.min(box1.y + box1.height, box2.y + box2.height);
+  if (x2 < x1 || y2 < y1) return 0;
+  const intersection = (x2 - x1) * (y2 - y1);
+  const area1 = box1.width * box1.height;
+  const area2 = box2.width * box2.height;
+  const union = area1 + area2 - intersection;
+  return intersection / union;
+}
+/**
+ * Get model status
+ * @returns {Object} Status information
+ */
+export function getManga109Status() {
+  return {
+    loaded: model !== null,
+    type: "YOLO26-nano (Manga109)",
+    accuracy: "95.6% mAP50",
+    classes: ["panel", "text"],
+  };
+}
+/**
+ * Clear cached model
+ */
+export function clearManga109Model() {
+  if (model) {
+    model.dispose();
+    model = null;
+  }
+  console.log("[Manga109] Model cache cleared");
+}
@@ -0,0 +1,253 @@
+// panel-detection/metadata-extractor.js
+// Extract panel coordinates from comic/manga metadata
+/**
+ * Extract panel regions from EPUB/HTML content
+ * @param {Document} doc - The document object from iframe
+ * @param {string} contentPath - Path to current content (for debugging)
+ * @returns {Promise<Object>} Result with panels array and metadata
+ */
+export async function extractPanelMetadata(doc, contentPath) {
+  console.log(`[Metadata Extraction] Extracting from: ${contentPath}`);
+
+  // Try Kodansha linkhotspots first (inline CSS - easy)
+  const linkhotspotPanels = extractLinkhotspots(doc);
+  if (linkhotspotPanels.length > 0) {
+    console.log(
+      `[Metadata Extraction] Found ${linkhotspotPanels.length} Kodansha linkhotspots`,
+    );
+    return {
+      panels: linkhotspotPanels,
+      source: "kodansha-linkhotspots",
+      confidence: 1.0,
+      complete: true, // Metadata is complete
+    };
+  }
+  // Try Amazon magnification regions (CSS-based - harder)
+  const amazonPanels = await extractAmazonRegions(doc);
+  if (amazonPanels.length > 0) {
+    console.log(
+      `[Metadata Extraction] Found ${amazonPanels.length} Amazon magnification regions`,
+    );
+    return {
+      panels: amazonPanels,
+      source: "amazon-magnify",
+      confidence: 1.0,
+      complete: true,
+    };
+  }
+  console.log("[Metadata Extraction] No panel metadata found");
+  return {
+    panels: [],
+    source: null,
+    confidence: 0,
+    complete: false,
+  };
+}
+/**
+ * Extract Kodansha-style linkhotspots with inline CSS coordinates
+ * @param {Document} doc - Document object
+ * @returns {Array} Array of panel objects
+ */
+function extractLinkhotspots(doc) {
+  const linkhotspots = doc.querySelectorAll('.linkhotspot[style*="top"]');
+  const panels = [];
+  linkhotspots.forEach((hotspot, index) => {
+    const style = hotspot.getAttribute("style");
+    if (!style) return;
+    const coords = parseInlineCSS(style);
+
+    if (coords && coords.top !== undefined) {
+      // Convert left/right to width, top/bottom to height
+      const width = 100 - coords.left - coords.right;
+      const height = 100 - coords.top - coords.bottom;
+
+      panels.push({
+        id: `metadata-linkhotspot-${index}`,
+        x: coords.left,
+        y: coords.top,
+        width: width,
+        height: height,
+        reading_order: index,
+        source: "kodansha-linkhotspot",
+        confidence: 1.0,
+      });
+    }
+  });
+  return panels;
+}
+/**
+ * Extract Amazon magnification regions using getBoundingClientRect()
+ * @param {Document} doc - Document object
+ * @returns {Promise<Array>} Array of panel objects
+ */
+async function extractAmazonRegions(doc) {
+  // Find all magnify links
+  const magnifyLinks = doc.querySelectorAll("a[data-app-amzn-magnify]");
+  if (magnifyLinks.length === 0) return [];
+  const panels = [];
+  const processed = new Set();
+  // Extract target IDs from data attributes
+  magnifyLinks.forEach((link, index) => {
+    try {
+      const data = JSON.parse(link.getAttribute("data-app-amzn-magnify"));
+      const targetId = data.targetId;
+
+      if (processed.has(targetId)) return;
+      processed.add(targetId);
+
+      // Find the target div
+      const targetDiv = doc.getElementById(targetId);
+      if (!targetDiv) {
+        console.warn(`[Metadata Extraction] Target not found: ${targetId}`);
+        return;
+      }
+      // Get computed position
+      const rect = targetDiv.getBoundingClientRect();
+      const parentRect = targetDiv.parentElement?.getBoundingClientRect();
+
+      console.log("[DEBUG] targetDiv:", targetDiv);
+      console.log("[DEBUG] targetDiv classes:", targetDiv.className);
+      console.log(
+        "[DEBUG] targetDiv getBoundingClientRect:",
+        targetDiv.getBoundingClientRect(),
+      );
+
+      if (!parentRect) {
+        console.warn(`[Metadata Extraction] No parent rect for: ${targetId}`);
+        return;
+      }
+      // Convert to percentages relative to parent
+      const x = ((rect.left - parentRect.left) / parentRect.width) * 100;
+      const y = ((rect.top - parentRect.top) / parentRect.height) * 100;
+      const width = (rect.width / parentRect.width) * 100;
+      const height = (rect.height / parentRect.height) * 100;
+      panels.push({
+        id: `metadata-amazon-${index}`,
+        x,
+        y,
+        width,
+        height,
+        reading_order: index,
+        source: "amazon-magnify",
+        confidence: 1.0,
+      });
+    } catch (e) {
+      console.error(`[Metadata Extraction] Failed to parse Amazon region:`, e);
+    }
+  });
+  return panels;
+}
+/**
+ * Parse inline CSS to extract coordinate percentages
+ * @param {string} style - Inline CSS string
+ * @returns {Object} Object with top, left, right, bottom
+ */
+function parseInlineCSS(style) {
+  const coords = {};
+
+  // Parse each property
+  const properties = style.split(";").map((s) => s.trim());
+
+  properties.forEach((prop) => {
+    const [key, value] = prop.split(":").map((s) => s.trim());
+    if (value && value.endsWith("%")) {
+      coords[key] = parseFloat(value);
+    }
+  });
+  return coords;
+}
+/**
+ * Check if document has extractable metadata
+ * @param {Document} doc - Document object
+ * @returns {boolean} True if metadata is present
+ */
+export function hasMetadata(doc) {
+  return (
+    doc.querySelectorAll('.linkhotspot[style*="top"]').length > 0 ||
+    doc.querySelectorAll("a[data-app-amzn-magnify]").length > 0
+  );
+}
+
+/**
+ * Extract Amazon regions by parsing CSS classes
+ * @param {Document} doc - Document object
+ * @returns {Array} Array of panel objects
+ */
+async function extractAmazonRegionsFromCSS(doc) {
+  const panels = [];
+
+  // Find all links that point to magnification targets
+  const magnifyLinks = doc.querySelectorAll("a[data-app-amzn-magnify]");
+  if (magnifyLinks.length === 0) return [];
+  // Get all stylesheets
+  const styleSheets = doc.styleSheets;
+  const cssRules = {};
+  // Build a map of class names to their computed styles
+  for (const sheet of styleSheets) {
+    try {
+      for (const rule of sheet.cssRules) {
+        if (rule.selectorText && rule.selectorText.startsWith(".")) {
+          const className = rule.selectorText.substring(1); // Remove '.'
+          cssRules[className] = rule.style;
+        }
+      }
+    } catch (e) {
+      // CORS may block access to some stylesheets
+      console.warn("[Metadata Extraction] Could not read stylesheet:", e);
+    }
+  }
+  // For each magnify link, find the target div and extract coordinates
+  const processed = new Set();
+  for (const link of magnifyLinks) {
+    try {
+      const data = JSON.parse(link.getAttribute("data-app-amzn-magnify"));
+      const targetId = data.targetId;
+      if (processed.has(targetId)) continue;
+      processed.add(targetId);
+      // Find the target div
+      const targetDiv = doc.getElementById(targetId);
+      if (!targetDiv) continue;
+      // Find the child img element
+      const img = targetDiv.querySelector("img");
+      if (!img) continue;
+      // Get the class that defines the panel region
+      // The panel region class is usually applied to a sibling or parent
+      // Look for div elements with percentage-based positioning
+      const siblings = targetDiv.parentElement?.querySelectorAll(
+        'div[class*="calibre"]',
+      );
+
+      for (const sibling of siblings) {
+        if (sibling.id === targetId) continue;
+
+        const style = sibling.getAttribute("style") || sibling.className;
+
+        // Try to extract coordinates from inline style or class
+        if (style && typeof style === "string") {
+          const coords = parseInlineCSS(style);
+
+          if (coords && coords.top !== undefined) {
+            // Found a panel region
+            const width = 100 - (coords.left || 0) - (coords.right || 0);
+            const height = 100 - (coords.top || 0) - (coords.bottom || 0);
+
+            panels.push({
+              id: `metadata-amazon-${panels.length}`,
+              x: coords.left || 0,
+              y: coords.top || 0,
+              width: width || 100,
+              height: height || 100,
+              reading_order: panels.length,
+              source: "amazon-magnify",
+              confidence: 1.0,
+            });
+            break; // Use the first valid sibling
+          }
+        }
+      }
+    } catch (e) {
+      console.error(`[Metadata Extraction] Failed to parse Amazon region:`, e);
+    }
+  }
+  return panels;
+}