diff --git a/panel-detection/manga109-tflite-detector.js b/panel-detection/manga109-tflite-detector.js new file mode 100644 index 0000000..421585d --- /dev/null +++ b/panel-detection/manga109-tflite-detector.js @@ -0,0 +1,203 @@ +// panel-detection/manga109-tflite-detector.js +// Manga109 panel detection using TFLite +let tfliteModel = null; +/** + * Initialize the Manga109 TFLite model + * @returns {Promise} True if loaded successfully + */ +export async function initManga109Model() { + if (tfliteModel) { + console.log("[Manga109 TFLite] Using cached model"); + return true; + } + try { + console.log("[Manga109 TFLite] Loading model..."); + // Check if TFLite is available + if (typeof tflite === "undefined") { + throw new Error( + "TensorFlow.js TFLite not loaded. Include @tensorflow/tfjs-tflite", + ); + } + // Load the TFLite model directly - NO CONVERSION NEEDED! + tfliteModel = await tflite.loadTFLiteModel( + "./vendor/manga109/model.tflite", + ); + console.log("[Manga109 TFLite] Model loaded successfully"); + return true; + } catch (error) { + console.error("[Manga109 TFLite] Failed to load model:", error); + tfliteModel = null; + return false; + } +} +/** + * Detect panels using Manga109 TFLite model + * @param {ImageData} imageData - Image data from canvas + * @returns {Promise} Array of panel detections + */ +export async function detectManga109Panels(imageData) { + if (!tfliteModel) { + throw new Error("Model not loaded. Call initManga109Model() first."); + } + console.log("[Manga109 TFLite] Detecting panels..."); + try { + const imgWidth = imageData.width; + const imgHeight = imageData.height; + // Convert ImageData to tensor + const tensor = tf.browser.fromPixels(imageData); + // Preprocess: resize to 640x640 (YOLO26 input size) + const resized = tf.image.resizeBilinear(tensor, [640, 640]); + + // Normalize to 0-1 and adjust for TFLite model input requirements + // YOLO models typically expect RGB in [0, 255] range or [0, 1] + // We'll use [0, 1] range + const normalized = resized.div(255.0); + + // Add batch dimension: [1, 640, 640, 3] + const batched = normalized.expandDims(0); + // Run inference + const startTime = performance.now(); + const outputTensor = tfliteModel.predict(batched); + const endTime = performance.now(); + console.log( + `[Manga109 TFLite] Inference took ${(endTime - startTime).toFixed(2)}ms`, + ); + // Postprocess outputs + const panels = postprocessTFLiteOutputs(outputTensor, imgWidth, imgHeight); + console.log(`[Manga109 TFLite] Detected ${panels.length} panels`); + // Clean up tensors + tensor.dispose(); + resized.dispose(); + normalized.dispose(); + batched.dispose(); + return panels; + } catch (error) { + console.error("[Manga109 TFLite] Detection failed:", error); + throw error; + } +} +/** + * Postprocess TFLite YOLO outputs to panel detections + * @param {Tensor} outputTensor - Model output tensor + * @param {number} imgWidth - Original image width + * @param {number} imgHeight - Original image height + * @returns {Array} Array of panel objects + */ +function postprocessTFLiteOutputs(outputTensor, imgWidth, imgHeight) { + const panels = []; + const confidenceThreshold = 0.5; + // TFLite YOLO output format + // Typically: [batch, num_detections, 85] for 80 classes (COCO) + // For 2-class model (panel, text): [batch, num_detections, 6] + // where 6 = [x_center, y_center, width, height, confidence, class_id] + + const outputArray = outputTensor.dataSync(); + const shape = outputTensor.shape; + + console.log(`[Manga109 TFLite] Output shape: [${shape.join(", ")}]`); + const [batch, numDetections, numClasses] = shape; + // Parse detections + for (let i = 0; i < numDetections; i++) { + const offset = i * numClasses; + // Extract values (may need adjustment based on actual model output) + const centerX = outputArray[offset]; + const centerY = outputArray[offset + 1]; + const width = outputArray[offset + 2]; + const height = outputArray[offset + 3]; + const confidence = outputArray[offset + 4]; + const classId = Math.round(outputArray[offset + 5]); + // Only detect panels (class 0) + if (classId !== 0) continue; + + if (confidence < confidenceThreshold) continue; + // Convert from center coords to top-left + // YOLO outputs are typically normalized [0, 1] or in pixels + // Assuming normalized output: + const halfWidth = width / 2; + const halfHeight = height / 2; + const x = (centerX - halfWidth) * imgWidth; + const y = (centerY - halfHeight) * imgHeight; + const w = width * imgWidth; + const h = height * imgHeight; + // Convert to percentages + panels.push({ + id: `manga109-tflite-${panels.length}`, + x: (x / imgWidth) * 100, + y: (y / imgHeight) * 100, + width: (w / imgWidth) * 100, + height: (h / imgHeight) * 100, + confidence: confidence, + reading_order: panels.length, + }); + } + // Apply Non-Maximum Suppression + return applyNMS(panels); +} + +/** + * Apply Non-Maximum Suppression to remove duplicate detections + * @param {Array} panels - Array of panel detections + * @param {number} iouThreshold - IoU threshold for NMS + * @returns {Array} Filtered panels + */ +function applyNMS(panels, iouThreshold = 0.5) { + if (panels.length === 0) return panels; + // Sort by confidence (highest first) + panels.sort((a, b) => b.confidence - a.confidence); + const keep = []; + const suppressed = new Set(); + for (let i = 0; i < panels.length; i++) { + if (suppressed.has(i)) continue; + keep.push(panels[i]); + // Suppress overlapping boxes + for (let j = i + 1; j < panels.length; j++) { + if (suppressed.has(j)) continue; + const iou = calculateIoU(panels[i], panels[j]); + if (iou > iouThreshold) { + suppressed.add(j); + } + } + } + return keep.map((p, i) => ({ ...p, reading_order: i })); +} +/** + * Calculate Intersection over Union (IoU) + * @param {Object} box1 - First panel + * @param {Object} box2 - Second panel + * @returns {number} IoU value + */ +function calculateIoU(box1, box2) { + // Convert percentage coordinates to pixels for calculation + const x1 = Math.max(box1.x, box2.x); + const y1 = Math.max(box1.y, box2.y); + const x2 = Math.min(box1.x + box1.width, box2.x + box2.width); + const y2 = Math.min(box1.y + box1.height, box2.y + box2.height); + if (x2 < x1 || y2 < y1) return 0; + const intersection = (x2 - x1) * (y2 - y1); + const area1 = box1.width * box1.height; + const area2 = box2.width * box2.height; + const union = area1 + area2 - intersection; + return intersection / union; +} +/** + * Get model status + * @returns {Object} Status information + */ +export function getManga109Status() { + return { + loaded: model !== null, + type: "YOLO26-nano (Manga109)", + accuracy: "95.6% mAP50", + classes: ["panel", "text"], + }; +} +/** + * Clear cached model + */ +export function clearManga109Model() { + if (model) { + model.dispose(); + model = null; + } + console.log("[Manga109] Model cache cleared"); +} diff --git a/panel-detection/metadata-extractor.js b/panel-detection/metadata-extractor.js new file mode 100644 index 0000000..88792da --- /dev/null +++ b/panel-detection/metadata-extractor.js @@ -0,0 +1,253 @@ +// panel-detection/metadata-extractor.js +// Extract panel coordinates from comic/manga metadata +/** + * Extract panel regions from EPUB/HTML content + * @param {Document} doc - The document object from iframe + * @param {string} contentPath - Path to current content (for debugging) + * @returns {Promise} Result with panels array and metadata + */ +export async function extractPanelMetadata(doc, contentPath) { + console.log(`[Metadata Extraction] Extracting from: ${contentPath}`); + + // Try Kodansha linkhotspots first (inline CSS - easy) + const linkhotspotPanels = extractLinkhotspots(doc); + if (linkhotspotPanels.length > 0) { + console.log( + `[Metadata Extraction] Found ${linkhotspotPanels.length} Kodansha linkhotspots`, + ); + return { + panels: linkhotspotPanels, + source: "kodansha-linkhotspots", + confidence: 1.0, + complete: true, // Metadata is complete + }; + } + // Try Amazon magnification regions (CSS-based - harder) + const amazonPanels = await extractAmazonRegions(doc); + if (amazonPanels.length > 0) { + console.log( + `[Metadata Extraction] Found ${amazonPanels.length} Amazon magnification regions`, + ); + return { + panels: amazonPanels, + source: "amazon-magnify", + confidence: 1.0, + complete: true, + }; + } + console.log("[Metadata Extraction] No panel metadata found"); + return { + panels: [], + source: null, + confidence: 0, + complete: false, + }; +} +/** + * Extract Kodansha-style linkhotspots with inline CSS coordinates + * @param {Document} doc - Document object + * @returns {Array} Array of panel objects + */ +function extractLinkhotspots(doc) { + const linkhotspots = doc.querySelectorAll('.linkhotspot[style*="top"]'); + const panels = []; + linkhotspots.forEach((hotspot, index) => { + const style = hotspot.getAttribute("style"); + if (!style) return; + const coords = parseInlineCSS(style); + + if (coords && coords.top !== undefined) { + // Convert left/right to width, top/bottom to height + const width = 100 - coords.left - coords.right; + const height = 100 - coords.top - coords.bottom; + + panels.push({ + id: `metadata-linkhotspot-${index}`, + x: coords.left, + y: coords.top, + width: width, + height: height, + reading_order: index, + source: "kodansha-linkhotspot", + confidence: 1.0, + }); + } + }); + return panels; +} +/** + * Extract Amazon magnification regions using getBoundingClientRect() + * @param {Document} doc - Document object + * @returns {Promise} Array of panel objects + */ +async function extractAmazonRegions(doc) { + // Find all magnify links + const magnifyLinks = doc.querySelectorAll("a[data-app-amzn-magnify]"); + if (magnifyLinks.length === 0) return []; + const panels = []; + const processed = new Set(); + // Extract target IDs from data attributes + magnifyLinks.forEach((link, index) => { + try { + const data = JSON.parse(link.getAttribute("data-app-amzn-magnify")); + const targetId = data.targetId; + + if (processed.has(targetId)) return; + processed.add(targetId); + + // Find the target div + const targetDiv = doc.getElementById(targetId); + if (!targetDiv) { + console.warn(`[Metadata Extraction] Target not found: ${targetId}`); + return; + } + // Get computed position + const rect = targetDiv.getBoundingClientRect(); + const parentRect = targetDiv.parentElement?.getBoundingClientRect(); + + console.log("[DEBUG] targetDiv:", targetDiv); + console.log("[DEBUG] targetDiv classes:", targetDiv.className); + console.log( + "[DEBUG] targetDiv getBoundingClientRect:", + targetDiv.getBoundingClientRect(), + ); + + if (!parentRect) { + console.warn(`[Metadata Extraction] No parent rect for: ${targetId}`); + return; + } + // Convert to percentages relative to parent + const x = ((rect.left - parentRect.left) / parentRect.width) * 100; + const y = ((rect.top - parentRect.top) / parentRect.height) * 100; + const width = (rect.width / parentRect.width) * 100; + const height = (rect.height / parentRect.height) * 100; + panels.push({ + id: `metadata-amazon-${index}`, + x, + y, + width, + height, + reading_order: index, + source: "amazon-magnify", + confidence: 1.0, + }); + } catch (e) { + console.error(`[Metadata Extraction] Failed to parse Amazon region:`, e); + } + }); + return panels; +} +/** + * Parse inline CSS to extract coordinate percentages + * @param {string} style - Inline CSS string + * @returns {Object} Object with top, left, right, bottom + */ +function parseInlineCSS(style) { + const coords = {}; + + // Parse each property + const properties = style.split(";").map((s) => s.trim()); + + properties.forEach((prop) => { + const [key, value] = prop.split(":").map((s) => s.trim()); + if (value && value.endsWith("%")) { + coords[key] = parseFloat(value); + } + }); + return coords; +} +/** + * Check if document has extractable metadata + * @param {Document} doc - Document object + * @returns {boolean} True if metadata is present + */ +export function hasMetadata(doc) { + return ( + doc.querySelectorAll('.linkhotspot[style*="top"]').length > 0 || + doc.querySelectorAll("a[data-app-amzn-magnify]").length > 0 + ); +} + +/** + * Extract Amazon regions by parsing CSS classes + * @param {Document} doc - Document object + * @returns {Array} Array of panel objects + */ +async function extractAmazonRegionsFromCSS(doc) { + const panels = []; + + // Find all links that point to magnification targets + const magnifyLinks = doc.querySelectorAll("a[data-app-amzn-magnify]"); + if (magnifyLinks.length === 0) return []; + // Get all stylesheets + const styleSheets = doc.styleSheets; + const cssRules = {}; + // Build a map of class names to their computed styles + for (const sheet of styleSheets) { + try { + for (const rule of sheet.cssRules) { + if (rule.selectorText && rule.selectorText.startsWith(".")) { + const className = rule.selectorText.substring(1); // Remove '.' + cssRules[className] = rule.style; + } + } + } catch (e) { + // CORS may block access to some stylesheets + console.warn("[Metadata Extraction] Could not read stylesheet:", e); + } + } + // For each magnify link, find the target div and extract coordinates + const processed = new Set(); + for (const link of magnifyLinks) { + try { + const data = JSON.parse(link.getAttribute("data-app-amzn-magnify")); + const targetId = data.targetId; + if (processed.has(targetId)) continue; + processed.add(targetId); + // Find the target div + const targetDiv = doc.getElementById(targetId); + if (!targetDiv) continue; + // Find the child img element + const img = targetDiv.querySelector("img"); + if (!img) continue; + // Get the class that defines the panel region + // The panel region class is usually applied to a sibling or parent + // Look for div elements with percentage-based positioning + const siblings = targetDiv.parentElement?.querySelectorAll( + 'div[class*="calibre"]', + ); + + for (const sibling of siblings) { + if (sibling.id === targetId) continue; + + const style = sibling.getAttribute("style") || sibling.className; + + // Try to extract coordinates from inline style or class + if (style && typeof style === "string") { + const coords = parseInlineCSS(style); + + if (coords && coords.top !== undefined) { + // Found a panel region + const width = 100 - (coords.left || 0) - (coords.right || 0); + const height = 100 - (coords.top || 0) - (coords.bottom || 0); + + panels.push({ + id: `metadata-amazon-${panels.length}`, + x: coords.left || 0, + y: coords.top || 0, + width: width || 100, + height: height || 100, + reading_order: panels.length, + source: "amazon-magnify", + confidence: 1.0, + }); + break; // Use the first valid sibling + } + } + } + } catch (e) { + console.error(`[Metadata Extraction] Failed to parse Amazon region:`, e); + } + } + return panels; +}