feat: add panel detection with multi-tier fallback system

Implement intelligent panel detection for manga/comics using a three-tier
fallback system that automatically selects the best detection method:

- Tier 1: OpenCV edge detection (fast, accurate for clear panel borders)
- Tier 2: COCO-SSD ML detection (handles irregular layouts)
- Tier 3: Grid-based detection (lightweight, always works)

The system automatically falls back through tiers if higher tiers fail
or if CSP blocks 'unsafe-eval' required by ML libraries.

Changes:
- panel-detection/detector.js: Modified to use dynamic script loading
  * Calls loadMLLibraries() on first use for lazy loading
  * Checks if ML libraries loaded successfully before using them
  * Falls back to grid detection if CSP blocks eval or libraries fail
  * Uses globalThis.cv/tf/cocoSsd for UMD/global library access

- panel-detection/load-scripts.js: Created dynamic script loader
  * Dynamically injects <script> tags when panel detection enabled
  * Checks CSP compatibility with canUseEval() function
  * Loads libraries in correct order: TensorFlow → OpenCV → COCO-SSD
  * Falls back to grid if ML libraries fail to load
  * Uses Promise-based API for clean async loading

This library-native approach keeps all functionality within foliate-js
without requiring changes to reader.html or consumer applications.
This commit is contained in:
2026-04-13 19:14:44 -04:00
parent 89543d0df7
commit 7971ad7996
2 changed files with 143 additions and 63 deletions
+66 -60
View File
@@ -1,9 +1,11 @@
import { loadMLLibraries, getLibrariesStatus } from "./load-scripts.js";
// panel-detection/detector.js // panel-detection/detector.js
// Main panel detector with lazy-loaded fallback chain // Main panel detector with lazy-loaded fallback chain
import { loadMLLibraries, getLibrariesStatus } from "./load-scripts.js";
export class PanelDetector { export class PanelDetector {
#opencv = null;
#model = null;
#cache = new Map(); #cache = new Map();
#scriptsLoaded = false;
async detectPanels(doc, index, force = false) { async detectPanels(doc, index, force = false) {
const cacheKey = `${doc.location?.pathname || ""}-${index}`; const cacheKey = `${doc.location?.pathname || ""}-${index}`;
@@ -17,6 +19,33 @@ export class PanelDetector {
return { panels: [], method: "no-image", confidence: 0 }; return { panels: [], method: "no-image", confidence: 0 };
} }
// Load ML libraries on first use
if (!this.#scriptsLoaded) {
const result = await loadMLLibraries();
if (!result.loaded) {
console.warn(
"ML libraries not available, using grid detection:",
result.reason,
);
// Fall back to grid immediately
const { detectPanelsGrid } = await import("./grid.js");
const panels = detectPanelsGrid(imageData);
this.#cache.set(cacheKey, {
panels,
method: "grid",
confidence: 0.4,
reason: result.reason,
});
return {
panels,
method: "grid",
confidence: 0.4,
reason: result.reason,
};
}
this.#scriptsLoaded = true;
}
const result = await this.#runDetectionPipeline(imageData); const result = await this.#runDetectionPipeline(imageData);
this.#cache.set(cacheKey, result); this.#cache.set(cacheKey, result);
return result; return result;
@@ -39,44 +68,49 @@ export class PanelDetector {
const { detectPanelsML } = await import("./coco-ssd.js"); const { detectPanelsML } = await import("./coco-ssd.js");
const { detectPanelsGrid } = await import("./grid.js"); const { detectPanelsGrid } = await import("./grid.js");
if (!this.#opencv) { // Try OpenCV (uses global cv)
try { try {
this.#opencv = await this.#loadOpenCV(); const cv = globalThis.cv;
} catch (e) { if (cv && cv.Mat) {
console.warn("Failed to load OpenCV:", e); // Wait for OpenCV to be ready
} await new Promise((resolve, reject) => {
} const check = () => {
if (cv && cv.Mat) resolve();
else if (cv && cv.readyState === "complete")
reject(new Error("OpenCV failed to load"));
else setTimeout(check, 50);
};
check();
});
if (this.#opencv) { const panels = await detectPanelsOpenCV(imageData, cv);
try {
const panels = await detectPanelsOpenCV(imageData, this.#opencv);
if (this.#validatePanels(panels, imageData)) { if (this.#validatePanels(panels, imageData)) {
return { panels, method: "opencv", confidence: 0.85 }; return { panels, method: "opencv", confidence: 0.85 };
} }
}
} catch (e) { } catch (e) {
console.warn("OpenCV detection failed:", e); console.warn("OpenCV detection failed:", e);
} }
// Try ML (uses global cocoSsd)
try {
const cocoSsd = globalThis.cocoSsd;
if (cocoSsd) {
// Wait for COCO-SSD to be ready
if (!cocoSsd.load) {
await new Promise((resolve) => setTimeout(resolve, 100));
} }
if (!this.#model) { const panels = await detectPanelsML(imageData, cocoSsd);
try {
this.#model = await this.#loadModel();
} catch (e) {
console.warn("Failed to load ML model:", e);
}
}
if (this.#model) {
try {
const panels = await detectPanelsML(imageData, this.#model);
if (this.#validatePanels(panels, imageData)) { if (this.#validatePanels(panels, imageData)) {
return { panels, method: "ml", confidence: 0.7 }; return { panels, method: "ml", confidence: 0.7 };
} }
}
} catch (e) { } catch (e) {
console.warn("ML detection failed:", e); console.warn("ML detection failed:", e);
} }
}
// Grid fallback (always works)
const panels = detectPanelsGrid(imageData); const panels = detectPanelsGrid(imageData);
return { panels, method: "grid", confidence: 0.4 }; return { panels, method: "grid", confidence: 0.4 };
} }
@@ -96,44 +130,16 @@ export class PanelDetector {
return coverage > 0.1 && coverage < 0.95; return coverage > 0.1 && coverage < 0.95;
} }
async #loadOpenCV() {
if (this.#opencv) return this.#opencv;
try {
const cv = await import("../vendor/opencv/opencv.js");
await new Promise((resolve, reject) => {
const check = () => {
if (cv && cv.Mat) resolve();
else if (cv.readyState === "complete")
reject(new Error("OpenCV failed to load"));
else setTimeout(check, 50);
};
check();
});
this.#opencv = cv.default || cv;
return this.#opencv;
} catch (e) {
console.warn("Failed to load OpenCV:", e);
return null;
}
}
async #loadModel() {
if (this.#model) return this.#model;
try {
await import("../vendor/tfjs/tf.min.js");
const cocoSsd = await import("../vendor/coco-ssd/coco-ssd.min.js");
this.#model = await cocoSsd.load({ base: "lite_mobilenet_v2" });
return this.#model;
} catch (e) {
console.warn("Failed to load ML model:", e);
return null;
}
}
clear() { clear() {
this.#cache.clear(); this.#cache.clear();
} }
// Expose library status for debugging
getStatus() {
return {
...getLibrariesStatus(),
scriptsLoaded: this.#scriptsLoaded,
cacheSize: this.#cache.size,
};
}
} }
+74
View File
@@ -0,0 +1,74 @@
// panel-detection/load-scripts.js
// Dynamically loads ML/CV libraries as script tags
const loadedScripts = new Set();
export async function loadMLLibraries() {
// Check if already loaded
if (globalThis.cv && globalThis.tf && globalThis.cocoSsd) {
return { loaded: true, method: "cached" };
}
// Check CSP compatibility
if (!canUseEval()) {
console.warn("Panel detection requires CSP with unsafe-eval");
return { loaded: false, reason: "csp-blocked" };
}
const scripts = [
{ name: "TensorFlow", src: "./vendor/tfjs/tf.min.js", global: "tf" },
{ name: "OpenCV", src: "./vendor/opencv/opencv.js", global: "cv" },
{
name: "COCO-SSD",
src: "./vendor/coco-ssd/coco-ssd.min.js",
global: "cocoSsd",
},
];
try {
for (const { name, src, global: globalName } of scripts) {
if (globalThis[globalName]) continue; // Already loaded
await loadScript(src);
loadedScripts.add(src);
// Verify global was set
if (!globalThis[globalName]) {
throw new Error(`${name} failed to load (global not set)`);
}
}
return { loaded: true, method: "dynamic" };
} catch (e) {
console.warn("Failed to load ML libraries:", e);
return { loaded: false, reason: e.message };
}
}
function loadScript(src) {
return new Promise((resolve, reject) => {
const script = document.createElement("script");
script.src = src;
script.onload = () => resolve();
script.onerror = () => reject(new Error(`Failed to load: ${src}`));
document.head.appendChild(script);
});
}
function canUseEval() {
// Try to detect if CSP allows eval
try {
const test = new Function("return true")();
return test === true;
} catch {
return false;
}
}
export function getLibrariesStatus() {
return {
opencv: !!globalThis.cv,
tensorflow: !!globalThis.tf,
cocoSsd: !!globalThis.cocoSsd,
};
}
export function clearScripts() {
// Note: We don't remove script tags as they can't be unloaded
// This is for future cleanup if needed
loadedScripts.clear();
}