mirror of
https://github.com/john-okeefe/foliate-js.git
synced 2026-09-09 11:29:14 -04:00
feat: add panel detection with multi-tier fallback system
Implement intelligent panel detection for manga/comics using a three-tier fallback system that automatically selects the best detection method: - Tier 1: OpenCV edge detection (fast, accurate for clear panel borders) - Tier 2: COCO-SSD ML detection (handles irregular layouts) - Tier 3: Grid-based detection (lightweight, always works) The system automatically falls back through tiers if higher tiers fail or if CSP blocks 'unsafe-eval' required by ML libraries. Changes: - panel-detection/detector.js: Modified to use dynamic script loading * Calls loadMLLibraries() on first use for lazy loading * Checks if ML libraries loaded successfully before using them * Falls back to grid detection if CSP blocks eval or libraries fail * Uses globalThis.cv/tf/cocoSsd for UMD/global library access - panel-detection/load-scripts.js: Created dynamic script loader * Dynamically injects <script> tags when panel detection enabled * Checks CSP compatibility with canUseEval() function * Loads libraries in correct order: TensorFlow → OpenCV → COCO-SSD * Falls back to grid if ML libraries fail to load * Uses Promise-based API for clean async loading This library-native approach keeps all functionality within foliate-js without requiring changes to reader.html or consumer applications.
This commit is contained in:
+66
-60
@@ -1,9 +1,11 @@
|
|||||||
|
import { loadMLLibraries, getLibrariesStatus } from "./load-scripts.js";
|
||||||
|
|
||||||
// panel-detection/detector.js
|
// panel-detection/detector.js
|
||||||
// Main panel detector with lazy-loaded fallback chain
|
// Main panel detector with lazy-loaded fallback chain
|
||||||
|
import { loadMLLibraries, getLibrariesStatus } from "./load-scripts.js";
|
||||||
export class PanelDetector {
|
export class PanelDetector {
|
||||||
#opencv = null;
|
|
||||||
#model = null;
|
|
||||||
#cache = new Map();
|
#cache = new Map();
|
||||||
|
#scriptsLoaded = false;
|
||||||
|
|
||||||
async detectPanels(doc, index, force = false) {
|
async detectPanels(doc, index, force = false) {
|
||||||
const cacheKey = `${doc.location?.pathname || ""}-${index}`;
|
const cacheKey = `${doc.location?.pathname || ""}-${index}`;
|
||||||
@@ -17,6 +19,33 @@ export class PanelDetector {
|
|||||||
return { panels: [], method: "no-image", confidence: 0 };
|
return { panels: [], method: "no-image", confidence: 0 };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Load ML libraries on first use
|
||||||
|
if (!this.#scriptsLoaded) {
|
||||||
|
const result = await loadMLLibraries();
|
||||||
|
if (!result.loaded) {
|
||||||
|
console.warn(
|
||||||
|
"ML libraries not available, using grid detection:",
|
||||||
|
result.reason,
|
||||||
|
);
|
||||||
|
// Fall back to grid immediately
|
||||||
|
const { detectPanelsGrid } = await import("./grid.js");
|
||||||
|
const panels = detectPanelsGrid(imageData);
|
||||||
|
this.#cache.set(cacheKey, {
|
||||||
|
panels,
|
||||||
|
method: "grid",
|
||||||
|
confidence: 0.4,
|
||||||
|
reason: result.reason,
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
panels,
|
||||||
|
method: "grid",
|
||||||
|
confidence: 0.4,
|
||||||
|
reason: result.reason,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
this.#scriptsLoaded = true;
|
||||||
|
}
|
||||||
|
|
||||||
const result = await this.#runDetectionPipeline(imageData);
|
const result = await this.#runDetectionPipeline(imageData);
|
||||||
this.#cache.set(cacheKey, result);
|
this.#cache.set(cacheKey, result);
|
||||||
return result;
|
return result;
|
||||||
@@ -39,44 +68,49 @@ export class PanelDetector {
|
|||||||
const { detectPanelsML } = await import("./coco-ssd.js");
|
const { detectPanelsML } = await import("./coco-ssd.js");
|
||||||
const { detectPanelsGrid } = await import("./grid.js");
|
const { detectPanelsGrid } = await import("./grid.js");
|
||||||
|
|
||||||
if (!this.#opencv) {
|
// Try OpenCV (uses global cv)
|
||||||
try {
|
try {
|
||||||
this.#opencv = await this.#loadOpenCV();
|
const cv = globalThis.cv;
|
||||||
} catch (e) {
|
if (cv && cv.Mat) {
|
||||||
console.warn("Failed to load OpenCV:", e);
|
// Wait for OpenCV to be ready
|
||||||
}
|
await new Promise((resolve, reject) => {
|
||||||
}
|
const check = () => {
|
||||||
|
if (cv && cv.Mat) resolve();
|
||||||
|
else if (cv && cv.readyState === "complete")
|
||||||
|
reject(new Error("OpenCV failed to load"));
|
||||||
|
else setTimeout(check, 50);
|
||||||
|
};
|
||||||
|
check();
|
||||||
|
});
|
||||||
|
|
||||||
if (this.#opencv) {
|
const panels = await detectPanelsOpenCV(imageData, cv);
|
||||||
try {
|
|
||||||
const panels = await detectPanelsOpenCV(imageData, this.#opencv);
|
|
||||||
if (this.#validatePanels(panels, imageData)) {
|
if (this.#validatePanels(panels, imageData)) {
|
||||||
return { panels, method: "opencv", confidence: 0.85 };
|
return { panels, method: "opencv", confidence: 0.85 };
|
||||||
}
|
}
|
||||||
|
}
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
console.warn("OpenCV detection failed:", e);
|
console.warn("OpenCV detection failed:", e);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Try ML (uses global cocoSsd)
|
||||||
|
try {
|
||||||
|
const cocoSsd = globalThis.cocoSsd;
|
||||||
|
if (cocoSsd) {
|
||||||
|
// Wait for COCO-SSD to be ready
|
||||||
|
if (!cocoSsd.load) {
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 100));
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!this.#model) {
|
const panels = await detectPanelsML(imageData, cocoSsd);
|
||||||
try {
|
|
||||||
this.#model = await this.#loadModel();
|
|
||||||
} catch (e) {
|
|
||||||
console.warn("Failed to load ML model:", e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (this.#model) {
|
|
||||||
try {
|
|
||||||
const panels = await detectPanelsML(imageData, this.#model);
|
|
||||||
if (this.#validatePanels(panels, imageData)) {
|
if (this.#validatePanels(panels, imageData)) {
|
||||||
return { panels, method: "ml", confidence: 0.7 };
|
return { panels, method: "ml", confidence: 0.7 };
|
||||||
}
|
}
|
||||||
|
}
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
console.warn("ML detection failed:", e);
|
console.warn("ML detection failed:", e);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
|
// Grid fallback (always works)
|
||||||
const panels = detectPanelsGrid(imageData);
|
const panels = detectPanelsGrid(imageData);
|
||||||
return { panels, method: "grid", confidence: 0.4 };
|
return { panels, method: "grid", confidence: 0.4 };
|
||||||
}
|
}
|
||||||
@@ -96,44 +130,16 @@ export class PanelDetector {
|
|||||||
return coverage > 0.1 && coverage < 0.95;
|
return coverage > 0.1 && coverage < 0.95;
|
||||||
}
|
}
|
||||||
|
|
||||||
async #loadOpenCV() {
|
|
||||||
if (this.#opencv) return this.#opencv;
|
|
||||||
|
|
||||||
try {
|
|
||||||
const cv = await import("../vendor/opencv/opencv.js");
|
|
||||||
|
|
||||||
await new Promise((resolve, reject) => {
|
|
||||||
const check = () => {
|
|
||||||
if (cv && cv.Mat) resolve();
|
|
||||||
else if (cv.readyState === "complete")
|
|
||||||
reject(new Error("OpenCV failed to load"));
|
|
||||||
else setTimeout(check, 50);
|
|
||||||
};
|
|
||||||
check();
|
|
||||||
});
|
|
||||||
|
|
||||||
this.#opencv = cv.default || cv;
|
|
||||||
return this.#opencv;
|
|
||||||
} catch (e) {
|
|
||||||
console.warn("Failed to load OpenCV:", e);
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
async #loadModel() {
|
|
||||||
if (this.#model) return this.#model;
|
|
||||||
|
|
||||||
try {
|
|
||||||
await import("../vendor/tfjs/tf.min.js");
|
|
||||||
const cocoSsd = await import("../vendor/coco-ssd/coco-ssd.min.js");
|
|
||||||
this.#model = await cocoSsd.load({ base: "lite_mobilenet_v2" });
|
|
||||||
return this.#model;
|
|
||||||
} catch (e) {
|
|
||||||
console.warn("Failed to load ML model:", e);
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
clear() {
|
clear() {
|
||||||
this.#cache.clear();
|
this.#cache.clear();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Expose library status for debugging
|
||||||
|
getStatus() {
|
||||||
|
return {
|
||||||
|
...getLibrariesStatus(),
|
||||||
|
scriptsLoaded: this.#scriptsLoaded,
|
||||||
|
cacheSize: this.#cache.size,
|
||||||
|
};
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,74 @@
|
|||||||
|
// panel-detection/load-scripts.js
|
||||||
|
// Dynamically loads ML/CV libraries as script tags
|
||||||
|
const loadedScripts = new Set();
|
||||||
|
export async function loadMLLibraries() {
|
||||||
|
// Check if already loaded
|
||||||
|
if (globalThis.cv && globalThis.tf && globalThis.cocoSsd) {
|
||||||
|
return { loaded: true, method: "cached" };
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check CSP compatibility
|
||||||
|
if (!canUseEval()) {
|
||||||
|
console.warn("Panel detection requires CSP with unsafe-eval");
|
||||||
|
return { loaded: false, reason: "csp-blocked" };
|
||||||
|
}
|
||||||
|
|
||||||
|
const scripts = [
|
||||||
|
{ name: "TensorFlow", src: "./vendor/tfjs/tf.min.js", global: "tf" },
|
||||||
|
{ name: "OpenCV", src: "./vendor/opencv/opencv.js", global: "cv" },
|
||||||
|
{
|
||||||
|
name: "COCO-SSD",
|
||||||
|
src: "./vendor/coco-ssd/coco-ssd.min.js",
|
||||||
|
global: "cocoSsd",
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
try {
|
||||||
|
for (const { name, src, global: globalName } of scripts) {
|
||||||
|
if (globalThis[globalName]) continue; // Already loaded
|
||||||
|
|
||||||
|
await loadScript(src);
|
||||||
|
loadedScripts.add(src);
|
||||||
|
|
||||||
|
// Verify global was set
|
||||||
|
if (!globalThis[globalName]) {
|
||||||
|
throw new Error(`${name} failed to load (global not set)`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return { loaded: true, method: "dynamic" };
|
||||||
|
} catch (e) {
|
||||||
|
console.warn("Failed to load ML libraries:", e);
|
||||||
|
return { loaded: false, reason: e.message };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
function loadScript(src) {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const script = document.createElement("script");
|
||||||
|
script.src = src;
|
||||||
|
script.onload = () => resolve();
|
||||||
|
script.onerror = () => reject(new Error(`Failed to load: ${src}`));
|
||||||
|
document.head.appendChild(script);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
function canUseEval() {
|
||||||
|
// Try to detect if CSP allows eval
|
||||||
|
try {
|
||||||
|
const test = new Function("return true")();
|
||||||
|
return test === true;
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
export function getLibrariesStatus() {
|
||||||
|
return {
|
||||||
|
opencv: !!globalThis.cv,
|
||||||
|
tensorflow: !!globalThis.tf,
|
||||||
|
cocoSsd: !!globalThis.cocoSsd,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
export function clearScripts() {
|
||||||
|
// Note: We don't remove script tags as they can't be unloaded
|
||||||
|
// This is for future cleanup if needed
|
||||||
|
loadedScripts.clear();
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user