From ac699e71ee1d3dafcdcee9998cadf44cd8f82667 Mon Sep 17 00:00:00 2001 From: John O'Keefe Date: Sat, 3 Oct 2026 14:49:22 -0400 Subject: [PATCH] feat(rtff-pdb): RTF and PalmDOC plain-text support (v0.5.0) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit rtf.js: a minimal single-pass RTF tokenizer extracting document text with paragraph structure — groups, control words with signed params, \\'hh Windows-1252 escapes, \\uN unicode with \\ucN fallback skipping, known text destinations skipped whole (fonttbl/colortbl/stylesheet/ info/headers/pict/field/...), spec-required ignoring of unknown words. Format runs are intentionally not converted; RTF is read as styled text like .txt. pdb.js: the TEXt/REAd PalmDOC container (PalmDOC, TealDoc) — the PDB record reader and LZ77 decompressor are shared from mobi.js (both exported; upstream already carried the TODO for this path). Records decode as Windows-1252 and feed the plain-text pipeline (paragraph split, line join, ~150KB sectioning, Part N TOC). makeBook dispatch: content-based (TEXtREAd magic at offset 60; '{\\rtf' magic), after MOBI and before the text branch. Tests: node harness covering store + compressed PalmDOC, RTF paragraphs /escapes/unicode/hex/destination-skipping, makeBook routing; txt/docx regression suites pass. --- mobi.js | 6 +-- package.json | 2 +- pdb.js | 57 ++++++++++++++++++++ rtf.js | 149 +++++++++++++++++++++++++++++++++++++++++++++++++++ view.js | 8 +++ 5 files changed, 218 insertions(+), 4 deletions(-) create mode 100644 pdb.js create mode 100644 rtf.js diff --git a/mobi.js b/mobi.js index 20c77a8..3b454e0 100644 --- a/mobi.js +++ b/mobi.js @@ -180,7 +180,7 @@ const concatTypedArray3 = (a, b, c) => { } const decoder = new TextDecoder() -const getString = buffer => decoder.decode(buffer) +export const getString = buffer => decoder.decode(buffer) const getUint = buffer => { if (!buffer) return const l = buffer.byteLength @@ -226,7 +226,7 @@ const countUnsetEnd = x => { return count } -const decompressPalmDOC = array => { +export const decompressPalmDOC = array => { let output = [] for (let i = 0; i < array.length; i++) { const byte = array[i] @@ -483,7 +483,7 @@ export const isMOBI = async file => { return magic === 'BOOKMOBI'// || magic === 'TEXtREAd' } -class PDB { +export class PDB { #file #offsets pdb diff --git a/package.json b/package.json index 3212e14..7ab75dd 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "foliate-js", - "version": "0.3.1", + "version": "0.5.0", "description": "Render e-books in the browser", "repository": { "type": "git", diff --git a/pdb.js b/pdb.js new file mode 100644 index 0000000..f8819ae --- /dev/null +++ b/pdb.js @@ -0,0 +1,57 @@ +// Plain-text Palm database (.pdb) support: the "TEXt"/"REAd" PDB +// container (PalmDOC, also TealDoc/TealPaint-text databases) stores a +// text document as numbered records, each optionally compressed with +// the PalmDOC LZ77 variant (the same compression MOBI uses — the +// container machinery is shared with mobi.js). Records are decoded as +// Windows-1252 (Palm-era text predates UTF-8) and the resulting text is +// handed to the plain-text pipeline for paragraph splitting, joining, +// and sectioning. +// +// Limitations (by design, MVP): no HUFF/CDIC compression (record 0 +// compression type 17480 — that is MOBI territory, handled by mobi.js), +// no TealDoc anchor/link extensions, Windows-1252 decoding only. + +import { getString } from './mobi.js' +import { makeTextBook } from './text.js' + +const PDB_MAGIC_OFFSET = 60 // type (4) + creator (4) + +export const isPDB = async file => { + const magic = getString(await file.slice(PDB_MAGIC_OFFSET, PDB_MAGIC_OFFSET + 8).arrayBuffer()) + return magic === 'TEXtREAd' +} + +export const makePalmDocBook = async file => { + const { PDB, decompressPalmDOC } = await import('./mobi.js') + const pdb = new PDB() + await pdb.open(file) + // record 0 opens with the PalmDOC header: compression (2 bytes), + // unused (2), text length (4), num text records (2), record size (2), + // encryption (2) — followed by optional TealDoc data + const r0 = new Uint8Array(await pdb.loadRecord(0)) + const compression = (r0[0] << 8) | r0[1] + const numTextRecords = (r0[8] << 8) | r0[9] + + if (compression !== 1 && compression !== 2) + throw new Error(`Unsupported PalmDOC compression type ${compression}`) + const count = numTextRecords || pdb.pdb.numRecords - 1 + + const parts = [] + for (let i = 1; i <= count; i++) { + const buf = new Uint8Array(await pdb.loadRecord(i)) + parts.push(compression === 2 ? decompressPalmDOC(buf) : buf) + } + const total = parts.reduce((sum, p) => sum + p.length, 0) + const bytes = new Uint8Array(total) + for (let i = 0, offset = 0; i < parts.length; i++) { + bytes.set(parts[i], offset) + offset += parts[i].length + } + // Palm-era text is Windows-1252 (Latin-1 superset), not UTF-8 + const text = new TextDecoder('windows-1252').decode(bytes) + const name = (pdb.pdb.name || '').replace(/\0/g, '').trim() + const fileName = name || file.name.replace(/\.[^.]+$/, '') + + // reuse the plain-text pipeline: paragraph split, line join, sections + return makeTextBook(new File([text], `${fileName}.txt`, { type: 'text/plain' })) +} diff --git a/rtf.js b/rtf.js new file mode 100644 index 0000000..e839892 --- /dev/null +++ b/rtf.js @@ -0,0 +1,149 @@ +// Rich Text Format (.rtf) support: a minimal RTF tokenizer that +// extracts the document text (with paragraph structure) and hands it to +// the plain-text pipeline. Format runs (bold/italic/fonts/colors) are +// intentionally NOT converted — RTF is read here as styled text, the +// same treatment .txt gets. +// +// What is handled: groups, control words with optional signed numeric +// parameters, \'hh hex escapes (Windows-1252), \uN unicode with \ucN +// fallback skipping, \par/\line/\tab/page/section breaks, dashes, +// quotes, spaces, \\ \{ \} escapes, and destination groups that carry +// no readable text (fonttbl, colortbl, stylesheet, info, pict, object, +// field/fldinst, generator, list tables, rsid tables, headers/footers, +// index entries, bookmarks). Unknown control words are ignored (the +// spec's required behavior); unknown `\*` optional destinations are +// kept and their text read. +// +// Limitations (by design, MVP): tables flatten to plain text, no +// images, no list numbering, footnotes inline after their reference. + +import { makeTextBook } from './text.js' + +export const isRTF = async file => { + // `{\rtf` magic — content-based, like every makeBook dispatch + const head = new Uint8Array(await file.slice(0, 5).arrayBuffer()) + const magic = String.fromCharCode(...head) + return magic === '{\\rtf' +} + +const DESTINATIONS = new Set([ + 'fonttbl', 'colortbl', 'stylesheet', 'info', 'pict', 'object', + 'thicket', 'field', 'fldinst', 'fldalt', 'generator', 'xmlnstbl', + 'listtable', 'listoverridetable', 'rsidtbl', 'latentstyles', + 'datastore', 'themedata', 'colorschememapping', 'pgdsctbl', + 'header', 'footer', 'headerl', 'headerr', 'headerf', + 'footerl', 'footerr', 'footerf', 'ftnsep', 'ftnsepc', + 'aftnsep', 'aftnsepc', 'xe', 'tc', 'bkmkstart', 'bkmkend', + 'nonshppict', 'mmathPr', +]) + +const CP1252_EXTRA = { + 128: '€', 129: '\uFFFD', 130: '‚', 131: 'ƒ', 132: '„', 133: '…', + 134: '†', 135: '‡', 136: 'ˆ', 137: '‰', 138: 'Š', 139: '‹', + 140: 'Œ', 142: 'Ž', 145: '‘', 146: '’', 147: '“', 148: '”', + 149: '•', 150: '–', 151: '—', 152: '˜', 153: '™', 154: 'š', + 155: '›', 156: 'œ', 158: 'ž', 159: 'Ÿ', +} + +const decodeByte = code => CP1252_EXTRA[code] ?? String.fromCharCode(code) + +const WORD_PARAMS = { + par: '\n\n', + line: '\n', + tab: '\t', + page: '\n\n', + sect: '\n\n', + emdash: '—', + endash: '–', + emspace: ' ', + enspace: ' ', + qmspace: ' ', + bullet: '•', + lquote: '‘', + rquote: '’', + ldblquote: '“', + rdblquote: '”', + zwnj: '', + zwj: '', +} + +export const rtfToText = input => { + const text = typeof input === 'string' ? input : new TextDecoder('windows-1252').decode(input) + const n = text.length + let out = '' + let unicodeSkip = 1 + // per-group skipping flags; a group skips if it or any ancestor is + // a text destination + const skipStack = [] + + const skipping = () => skipStack.some(Boolean) + + let i = 0 + while (i < n) { + const ch = text[i] + if (ch === '\\') { + const next = text[i + 1] ?? '' + if (next === '\\' || next === '{' || next === '}') { + if (!skipping()) out += next + i += 2 + continue + } + if (next === "'") { + const hex = text.slice(i + 2, i + 4) + if (!skipping() && /^[0-9a-fA-F]{2}$/.test(hex)) + out += decodeByte(parseInt(hex, 16)) + i += 4 + continue + } + if (/[a-zA-Z]/.test(next)) { + // control word: letters, then an optional signed integer, + // terminated by a (consumed) space or any other character + let j = i + 1 + let word = '' + while (j < n && /[a-zA-Z]/.test(text[j])) word += text[j++] + let param = null + if (/[0-9-]/.test(text[j] ?? '')) { + let digits = '' + if (text[j] === '-') { digits = '-'; j++ } + while (j < n && /[0-9]/.test(text[j])) digits += text[j++] + param = parseInt(digits) + } + if (text[j] === ' ') j++ + i = j + // process + if (word === 'uc' && param != null) { + unicodeSkip = param + } else if (word === 'u' && param != null) { + let code = param < 0 ? param + 65536 : param + if (!skipping()) out += String.fromCharCode(code) + // skip the \ucN fallback characters after the escape + for (let k = 0; k < unicodeSkip; k++) { + if (text[i] === ' ') i++ + else if (text[i] === '\\' && text[i + 1] === "'") i += 4 + else i++ + } + } else if (DESTINATIONS.has(word)) { + if (skipStack.length) skipStack[skipStack.length - 1] = true + } else if (!skipping() && WORD_PARAMS[word]) { + out += WORD_PARAMS[word] + } + continue + } + // any other control symbol (\*, \~, \|-…) — ignored + i += 2 + continue + } + if (ch === '{') { skipStack.push(false); i++; continue } + if (ch === '}') { skipStack.pop(); i++; continue } + if (ch === '\r' || ch === '\n') { i++; continue } + if (!skipping()) out += ch + i++ + } + return out +} + +export const makeRTFBook = async file => { + const text = rtfToText(new Uint8Array(await file.arrayBuffer())) + if (!text.trim()) throw new Error('No extractable text in RTF') + return makeTextBook(new File([text], file.name.replace(/\.[^.]+$/, '') + '.txt', { type: 'text/plain' })) +} diff --git a/view.js b/view.js index 0041a18..3adada7 100644 --- a/view.js +++ b/view.js @@ -121,6 +121,14 @@ export const makeBook = async (file, options = {}) => { const { makeFB2 } = await import('./fb2.js') book = await makeFB2(file) } + else if (await (await import('./pdb.js')).isPDB(file)) { + const { makePalmDocBook } = await import('./pdb.js') + book = await makePalmDocBook(file) + } + else if (await (await import('./rtf.js')).isRTF(file)) { + const { makeRTFBook } = await import('./rtf.js') + book = await makeRTFBook(file) + } else { const { isText, makeTextBook } = await import('./text.js') if (isText(file)) book = await makeTextBook(file)