Files
bookhoard.koplugin/main.lua
T
john-okeefe a0ab354763 Paint applied highlights natively (yellow) and immediately; suppress echo color
Two rendering observations from live use:

- Synced highlights rendered grey while device-native ones are yellow:
  KOReader natively stamps view.highlight.saved_color ("yellow" on
  color screens) onto every new highlight; a nil color takes the
  darkenRect path (grey). Applied entries now carry the device's own
  saved_color, so they look exactly like native ones.

- They only appeared after a restart: ReaderView caches each rendered
  page's highlight boxes and only invalidates the cache on
  AnnotationsModified — a plain setDirty repaints from the stale
  cache. applyServerAnnotations now dispatches AnnotationsModified
  with a non-table payload (clears the whole cache) before setDirty.

Echo safety for the new default color: an applied entry's yellow is
the device default, not user intent — pushing it back would clobber
the web color. collectAnnotations suppresses the color for entries we
applied (bookhoard_dedup_key set) that the user has not modified
since (datetime_updated, which KOReader sets on any edit); edited
entries push their color and the server lets the edit win. Verified
in the harness: applied entry gets color=yellow; un-edited echo
carries no color; an edited (datetime_updated + green) echo sends
green and the web row updates; dedup keys round-trip throughout.
2026-08-19 20:25:10 -04:00

1527 lines
52 KiB
Lua

local ConfirmBox = require("ui/widget/confirmbox")
local DataStorage = require("datastorage")
local Device = require("device")
local Event = require("ui/event")
local InfoMessage = require("ui/widget/infomessage")
local InputDialog = require("ui/widget/inputdialog")
local LuaSettings = require("luasettings")
local Math = require("optmath")
local NetworkMgr = require("ui/network/manager")
local SpinWidget = require("ui/widget/spinwidget")
local UIManager = require("ui/uimanager")
local WidgetContainer = require("ui/widget/container/widgetcontainer")
local json = require("json")
local json_util = require("json.util")
local logger = require("logger")
local sha2 = require("ffi/sha2")
local time = require("ui/time")
local util = require("util")
local lfs = require("libs/libkoreader-lfs")
local T = require("ffi/util").template
local _ = require("gettext")
local BookhoardAPI = require("BookhoardAPI")
local SYNC_STRATEGY = {
PROMPT = 1,
SILENT = 2,
DISABLE = 3,
}
local SYNC_MODE = {
IMMEDIATE = "immediate",
CHECKPOINT = "checkpoint",
}
local API_CALL_DEBOUNCE_DELAY = time.s(25)
local PERIODIC_PUSH_DELAY = 10
local sha256hex = sha2.sha256hex or sha2.sha256
local Bookhoard = WidgetContainer:extend({
name = "bookhoard",
is_doc_only = false,
title = _("Bookhoard Server"),
settings_key = "bookhoard",
push_timestamp = nil,
pull_timestamp = nil,
page_update_counter = nil,
last_page = nil,
periodic_push_task = nil,
periodic_push_scheduled = nil,
registration_poll_scheduled = nil,
settings = nil,
})
Bookhoard.default_settings = {
server_url = nil,
auth_token = nil,
device_id = nil,
auto_sync = false,
pages_before_update = 50,
sync_forward = SYNC_STRATEGY.PROMPT,
sync_backward = SYNC_STRATEGY.DISABLE,
sync_progress = true,
sync_bookmarks = true,
sync_highlights = true,
sync_notes = true,
sync_mode = SYNC_MODE.IMMEDIATE,
}
function Bookhoard:init()
self.push_timestamp = 0
self.pull_timestamp = 0
self.page_update_counter = 0
self.last_page = -1
self.periodic_push_scheduled = false
self.registration_poll_scheduled = false
self.periodic_push_task = function()
self.periodic_push_scheduled = false
self.page_update_counter = 0
self:updateProgress(false, false)
end
-- readSetting returns the STORED table verbatim when one exists — it
-- does not merge defaults. A settings file saved by an older plugin
-- version lacks the annotation toggles entirely, so sync_bookmarks/
-- highlights/notes read as nil, the "or" in _doUpdateProgress fell
-- through to the else branch, and every push carried EMPTY annotation
-- arrays while progress synced fine. Merge missing defaults (and
-- persist once so the file and the menu checkboxes self-heal).
self.settings = G_reader_settings:readSetting(self.settings_key, {})
if type(self.settings) ~= "table" then
self.settings = {}
end
local settings_merged = false
for key, value in pairs(self.default_settings) do
if self.settings[key] == nil then
self.settings[key] = value
settings_merged = true
end
end
if settings_merged then
G_reader_settings:saveSetting(self.settings_key, self.settings)
logger.info("Bookhoard: merged new default settings (older settings file)")
end
if self.settings.auto_sync
and Device:hasSeamlessWifiToggle()
and G_reader_settings:readSetting("wifi_enable_action") ~= "turn_on" then
self.settings.auto_sync = false
logger.warn("Bookhoard: auto-sync disabled because wifi_enable_action is not turn_on")
end
self.ui.menu:registerToMainMenu(self)
self:setupMenuOrder()
end
function Bookhoard:onReaderReady()
if self.settings.auto_sync then
UIManager:nextTick(function()
-- On the first open of a freshly downloaded book there is no cached
-- bookhoard UUID yet (it is only learned from a successful push
-- response). Pulling now would just fail with "Push progress first".
-- Instead, push once to bootstrap identity: the server resolves the
-- book by SHA-256 (format-aware) and returns the UUID, which we then
-- cache. After that, push and pull both work without ordering.
if not self:getBookhoardUUID() then
self:updateProgress(true, false)
else
self:getProgress(true, false)
end
end)
end
self:registerEvents()
self.last_page = self.ui:getCurrentPage()
end
function Bookhoard:registerEvents()
if self.settings.auto_sync then
self.onCloseDocument = self._onCloseDocument
self.onPageUpdate = self._onPageUpdate
self.onResume = self._onResume
self.onSuspend = self._onSuspend
self.onNetworkConnected = self._onNetworkConnected
self.onNetworkDisconnecting = self._onNetworkDisconnecting
else
self.onCloseDocument = nil
self.onPageUpdate = nil
self.onResume = nil
self.onSuspend = nil
self.onNetworkConnected = nil
self.onNetworkDisconnecting = nil
end
end
function Bookhoard:getAPI()
return BookhoardAPI:new({
server_url = self.settings.server_url,
auth_token = self.settings.auth_token,
})
end
function Bookhoard:isConfigured()
return self.settings.server_url and self.settings.auth_token
end
function Bookhoard:getSyncPeriod()
if not self.settings.auto_sync then
return _("Not available")
end
local period = self.settings.pages_before_update
if period and period > 0 then
return period
end
return _("Never")
end
function Bookhoard:addToMainMenu(menu_items)
menu_items.bookhoard_sync = {
text = _("Bookhoard sync"),
sorting_hint = "tools",
sub_item_table = self:buildMainMenu(),
}
end
function Bookhoard:buildMainMenu()
local items = {}
if self:isConfigured() then
table.insert(items, {
text = _("Push progress from this device"),
callback = function()
self:updateProgress(true, true)
end,
})
table.insert(items, {
text = _("Pull progress from server"),
callback = function()
self:getProgress(true, true)
end,
separator = true,
})
end
table.insert(items, {
text = _("Server URL"),
keep_menu_open = true,
tap_input_func = function()
return {
title = _("Bookhoard server URL"),
input = self.settings.server_url or "http://",
callback = function(input)
self.settings.server_url = input ~= "" and input or nil
G_reader_settings:saveSetting(self.settings_key, self.settings)
end,
}
end,
})
if self:isConfigured() then
table.insert(items, {
text = _("Device info"),
keep_menu_open = true,
callback = function()
UIManager:show(InfoMessage:new{
text = T(_("Device ID: %1\nServer: %2"),
self.settings.device_id or _("unknown"),
self.settings.server_url),
})
end,
})
table.insert(items, {
text = _("Disconnect"),
keep_menu_open = true,
callback = function()
UIManager:show(ConfirmBox:new{
text = _("Disconnect from Bookhoard server?"),
ok_text = _("Disconnect"),
ok_callback = function()
self.settings.auth_token = nil
self.settings.device_id = nil
self.settings.auto_sync = false
G_reader_settings:saveSetting(self.settings_key, self.settings)
self:registerEvents()
UIManager:askForRestart()
end,
})
end,
separator = true,
})
else
table.insert(items, {
text = _("Register device"),
keep_menu_open = true,
callback = function()
self:startRegistration()
end,
separator = true,
})
end
table.insert(items, {
text = _("Automatically push progress"),
checked_func = function() return self.settings.auto_sync end,
help_text = _([[This may lead to prompts about toggling WiFi on document close and suspend/resume, depending on your device's connectivity.]]),
callback = function()
self:toggleAutoSync()
end,
})
table.insert(items, {
text_func = function()
return T(_("Periodically sync every # pages (%1)"), self:getSyncPeriod())
end,
enabled_func = function() return self.settings.auto_sync end,
keep_menu_open = true,
callback = function(touchmenu_instance)
local spin = SpinWidget:new{
text = _([[Number of page turns between progress updates. Set to 0 to disable.]]),
value = self.settings.pages_before_update or 0,
value_min = 0,
value_max = 999,
value_step = 1,
value_hold_step = 10,
ok_text = _("Set"),
title_text = _("Pages before update"),
default_value = 50,
callback = function(spin)
self.settings.pages_before_update = spin.value > 0 and spin.value or nil
G_reader_settings:saveSetting(self.settings_key, self.settings)
if touchmenu_instance then touchmenu_instance:updateItems() end
end,
}
UIManager:show(spin)
end,
})
table.insert(items, {
text_func = function()
return T(_("Sync mode (%1)"),
self.settings.sync_mode == SYNC_MODE.IMMEDIATE and _("immediate") or _("checkpoint"))
end,
sub_item_table = {
{
text = _("Immediate"),
checked_func = function()
return self.settings.sync_mode == SYNC_MODE.IMMEDIATE
end,
callback = function()
self.settings.sync_mode = SYNC_MODE.IMMEDIATE
G_reader_settings:saveSetting(self.settings_key, self.settings)
end,
},
{
text = _("Checkpoint"),
checked_func = function()
return self.settings.sync_mode == SYNC_MODE.CHECKPOINT
end,
callback = function()
self.settings.sync_mode = SYNC_MODE.CHECKPOINT
G_reader_settings:saveSetting(self.settings_key, self.settings)
end,
},
},
separator = true,
})
table.insert(items, {
text = _("Sync behavior"),
sub_item_table = {
{
text_func = function()
return T(_("Sync to a newer state (%1)"),
self:getStrategyName(self.settings.sync_forward))
end,
sub_item_table = {
{
text = _("Silently"),
checked_func = function()
return self.settings.sync_forward == SYNC_STRATEGY.SILENT
end,
callback = function()
self.settings.sync_forward = SYNC_STRATEGY.SILENT
G_reader_settings:saveSetting(self.settings_key, self.settings)
end,
},
{
text = _("Prompt"),
checked_func = function()
return self.settings.sync_forward == SYNC_STRATEGY.PROMPT
end,
callback = function()
self.settings.sync_forward = SYNC_STRATEGY.PROMPT
G_reader_settings:saveSetting(self.settings_key, self.settings)
end,
},
{
text = _("Never"),
checked_func = function()
return self.settings.sync_forward == SYNC_STRATEGY.DISABLE
end,
callback = function()
self.settings.sync_forward = SYNC_STRATEGY.DISABLE
G_reader_settings:saveSetting(self.settings_key, self.settings)
end,
},
},
},
{
text_func = function()
return T(_("Sync to an older state (%1)"),
self:getStrategyName(self.settings.sync_backward))
end,
sub_item_table = {
{
text = _("Silently"),
checked_func = function()
return self.settings.sync_backward == SYNC_STRATEGY.SILENT
end,
callback = function()
self.settings.sync_backward = SYNC_STRATEGY.SILENT
G_reader_settings:saveSetting(self.settings_key, self.settings)
end,
},
{
text = _("Prompt"),
checked_func = function()
return self.settings.sync_backward == SYNC_STRATEGY.PROMPT
end,
callback = function()
self.settings.sync_backward = SYNC_STRATEGY.PROMPT
G_reader_settings:saveSetting(self.settings_key, self.settings)
end,
},
{
text = _("Never"),
checked_func = function()
return self.settings.sync_backward == SYNC_STRATEGY.DISABLE
end,
callback = function()
self.settings.sync_backward = SYNC_STRATEGY.DISABLE
G_reader_settings:saveSetting(self.settings_key, self.settings)
end,
},
},
},
},
separator = true,
})
table.insert(items, {
text = _("What to sync"),
sub_item_table = {
{
text = _("Reading progress"),
checked_func = function() return self.settings.sync_progress end,
callback = function()
self.settings.sync_progress = not self.settings.sync_progress
G_reader_settings:saveSetting(self.settings_key, self.settings)
end,
},
{
text = _("Bookmarks"),
checked_func = function() return self.settings.sync_bookmarks end,
callback = function()
self.settings.sync_bookmarks = not self.settings.sync_bookmarks
G_reader_settings:saveSetting(self.settings_key, self.settings)
end,
},
{
text = _("Highlights"),
checked_func = function() return self.settings.sync_highlights end,
callback = function()
self.settings.sync_highlights = not self.settings.sync_highlights
G_reader_settings:saveSetting(self.settings_key, self.settings)
end,
},
{
text = _("Notes"),
checked_func = function() return self.settings.sync_notes end,
callback = function()
self.settings.sync_notes = not self.settings.sync_notes
G_reader_settings:saveSetting(self.settings_key, self.settings)
end,
},
},
separator = true,
})
table.insert(items, {
text = _("Setup OPDS catalog"),
keep_menu_open = true,
enabled_func = function() return self:isConfigured() end,
callback = function()
if self:setupOPDS() then
UIManager:show(InfoMessage:new{
text = _("Bookhoard OPDS catalog added! Find it in Home → OPDS Catalog."),
timeout = 3,
})
else
UIManager:show(InfoMessage:new{
text = _("Please configure and register your device first."),
timeout = 3,
})
end
end,
})
return items
end
function Bookhoard:getStrategyName(strategy)
if strategy == SYNC_STRATEGY.PROMPT then
return _("Prompt")
elseif strategy == SYNC_STRATEGY.SILENT then
return _("Auto")
else
return _("Disable")
end
end
function Bookhoard:toggleAutoSync()
if not self.settings.auto_sync
and Device:hasSeamlessWifiToggle()
and G_reader_settings:readSetting("wifi_enable_action") ~= "turn_on" then
UIManager:show(InfoMessage:new{
text = _("Set 'Action when Wi-Fi is off' to 'turn on' in Network settings to enable auto sync."),
})
return
end
self.settings.auto_sync = not self.settings.auto_sync
self:registerEvents()
G_reader_settings:saveSetting(self.settings_key, self.settings)
if self.settings.auto_sync and self.ui.doc_settings then
self:getProgress(true, true)
end
end
function Bookhoard:setupMenuOrder()
local settings_dir = DataStorage:getSettingsDir()
self:patchMenuOrderFile(settings_dir .. "/reader_menu_order.lua")
self:patchMenuOrderFile(settings_dir .. "/filemanager_menu_order.lua")
end
function Bookhoard:patchMenuOrderFile(filepath)
local default_tools = {
"read_timer",
"calibre",
"bookhoard_sync",
"exporter",
"statistics",
"progress_sync",
"move_to_archive",
"wallabag",
"news_downloader",
"text_editor",
"profiles",
"qrclipboard",
"----------------------------",
"more_tools",
}
local attr = lfs.attributes(filepath)
if not attr then
local f = io.open(filepath, "w")
if not f then return end
f:write("return {\n tools = {\n")
for _, id in ipairs(default_tools) do
f:write(' "' .. id .. '",\n')
end
f:write(" },\n}\n")
f:close()
logger.dbg("Bookhoard: created menu order file", filepath)
return
end
local existing = dofile(filepath)
if not existing or type(existing) ~= "table" then return end
local tools = existing.tools
if not tools or type(tools) ~= "table" then return end
for _, id in ipairs(tools) do
if id == "bookhoard" then return end
end
local insert_pos = nil
for i, id in ipairs(tools) do
if id == "calibre" then
insert_pos = i + 1
break
end
end
if not insert_pos then
insert_pos = 2
end
table.insert(tools, insert_pos, "bookhoard")
existing.tools = tools
local f = io.open(filepath, "w")
if not f then return end
f:write("return {\n")
for key, val in pairs(existing) do
if type(val) == "table" then
f:write(" " .. key .. " = {\n")
for _, id in ipairs(val) do
f:write(' "' .. id .. '",\n')
end
f:write(" },\n")
end
end
f:write("}\n")
f:close()
logger.dbg("Bookhoard: patched menu order file", filepath)
end
function Bookhoard:startRegistration()
if not self.settings.server_url or self.settings.server_url == "" then
UIManager:show(InfoMessage:new{
text = _("Please set your server URL first."),
timeout = 3,
})
return
end
if NetworkMgr:willRerunWhenOnline(function() self:startRegistration() end) then
return
end
local device_name = Device.model or "KOReader Device"
local device_identifier = Device:info() or device_name
UIManager:show(InfoMessage:new{
text = _("Registering device…"),
timeout = 1,
})
UIManager:scheduleIn(0.5, function()
local api = BookhoardAPI:new({ server_url = self.settings.server_url })
local ok, result = api:registerDevice(device_name, device_identifier)
if not ok then
UIManager:show(InfoMessage:new{
text = T(_("Registration failed: %1"),
result and result.error or _("unknown error")),
})
return
end
self.registration_id = result.registration_id
self.waiting_dialog = InfoMessage:new{
text = T(_("Device registered on server.\n\nOpen your Bookhoard web UI and go to:\n%1/devices\n\nApprove this device in the \"Pending Device Registrations\" section.\n\nWaiting for approval…"), self.settings.server_url),
}
UIManager:show(self.waiting_dialog)
self:startRegistrationPoll()
end)
end
function Bookhoard:startRegistrationPoll()
if self.registration_poll_scheduled then return end
self.registration_poll_scheduled = true
local function poll()
self.registration_poll_scheduled = false
if not self.registration_id then return end
local api = BookhoardAPI:new({ server_url = self.settings.server_url })
local ok, result = api:checkRegistrationStatus(self.registration_id)
if not ok then
if result and result.status == 410 then
self.registration_id = nil
UIManager:show(InfoMessage:new{
text = _("Registration expired. Please try again."),
})
return
end
self.registration_poll_scheduled = true
UIManager:scheduleIn(3, poll)
return
end
if result.status == "approved" then
self.registration_id = nil
self.settings.auth_token = result.auth_token
self.settings.device_id = result.device_id and tostring(result.device_id) or nil
G_reader_settings:saveSetting(self.settings_key, self.settings)
self:registerEvents()
self:setupOPDS()
if self.waiting_dialog then
UIManager:close(self.waiting_dialog)
self.waiting_dialog = nil
end
UIManager:show(InfoMessage:new{
text = _("Device registered successfully!"),
timeout = 3,
})
elseif result.status == "pending" then
self.registration_poll_scheduled = true
UIManager:scheduleIn(3, poll)
else
self.registration_id = nil
UIManager:show(InfoMessage:new{
text = T(_("Registration %1"), result.status or _("failed")),
})
end
end
UIManager:scheduleIn(3, poll)
end
function Bookhoard:setupOPDS()
if not self.settings.server_url or not self.settings.device_id
or not self.settings.auth_token then
return false
end
local opds_base_url = self.settings.server_url
.. "/opds/devices/" .. self.settings.device_id .. "/catalog"
local opds_url = opds_base_url .. "?token=" .. self.settings.auth_token
local opds_settings_file = DataStorage:getSettingsDir() .. "/opds.lua"
local opds_settings = LuaSettings:open(opds_settings_file)
local servers = opds_settings:readSetting("servers", {})
local found = false
for i, server in ipairs(servers) do
if server.url and (
server.url == opds_url
or server.url == opds_base_url
or server.url:sub(1, #opds_base_url + 1) == opds_base_url .. "?"
) then
servers[i] = {
title = "Bookhoard",
url = opds_url,
}
found = true
break
end
end
if not found then
table.insert(servers, {
title = "Bookhoard",
url = opds_url,
})
end
opds_settings:saveSetting("servers", servers)
opds_settings:flush()
if self.ui.opds then
self.ui.opds.servers = servers
end
return true
end
function Bookhoard:getLastPercent()
if self.ui.document.info.has_pages then
return Math.roundPercent(self.ui.paging:getLastPercent())
else
return Math.roundPercent(self.ui.rolling:getLastPercent())
end
end
function Bookhoard:getLastProgress()
if self.ui.document.info.has_pages then
return self.ui.paging:getLastProgress()
else
return self.ui.rolling:getLastProgress()
end
end
function Bookhoard:getFileSHA256()
local cached = self.ui.doc_settings:readSetting("bookhoard_sha256")
if cached then return cached end
local file = io.open(self.ui.document.file, "rb")
if not file then return nil end
local data = file:read("*a")
file:close()
local hash = sha256hex(data)
if hash then
self.ui.doc_settings:saveSetting("bookhoard_sha256", hash)
end
return hash
end
function Bookhoard:getBookhoardUUID()
if not self.ui.doc_settings then return nil end
return self.ui.doc_settings:readSetting("bookhoard_uuid")
end
function Bookhoard:getContextText()
if not self.ui.document or self.ui.document.info.has_pages then
return ""
end
local xp = self:getLastProgress()
if not xp then return "" end
local text = self.ui.document:getTextFromXPointer(xp)
if not text or text == "" then
text = self.ui.document:getTextFromXPointer(self.ui.document:getNormalizedXPointer(xp))
end
if not text or text == "" then return "" end
local char_offset = tonumber(xp:match("text%(%)%.?(%d+)")) or 0
if char_offset > 0 and char_offset < #text then
text = text:sub(char_offset + 1)
end
if #text > 100 then
text = text:sub(1, 100)
end
text = text:gsub("%s+", " "):match("^%s*(.-)%s*$") or ""
return text
end
function Bookhoard:collectBookData()
local props = self.ui.doc_props
local file_path = self.ui.document.file
local title = props.display_title or ""
local authors = {}
if props.authors then
authors = { props.authors }
end
json_util.InitArray(authors)
local file_sha256 = self:getFileSHA256()
local book_uuid = self:getBookhoardUUID()
local percentage = self:getLastPercent()
local progress = self:getLastProgress()
local page = self.ui:getCurrentPage()
local total_pages = self.ui.document:getPageCount()
local context_text = self:getContextText()
local book_data = {
uuid = book_uuid,
sha256 = file_sha256,
title = title,
authors = authors,
percentage = percentage,
context_text = context_text,
page = page,
total_pages = total_pages,
file_path = file_path,
device_info = {
koreader_version = require("version"):getCurrentRevision(),
device_model = Device.model,
},
}
-- epubcfi (a CREngine xpointer) is only meaningful for reflowable (rolling)
-- documents. Paging docs (PDF/comics/DjVu) carry their position in
-- page/total_pages; a bare page number here would be a JSON number into a
-- server *string field and fail the bind. Reflowable EPUBs are always
-- rolling, so they keep sending the xpointer exactly as before.
if not self.ui.document.info.has_pages then
book_data.epubcfi = progress
end
return book_data
end
-- KOReader stores annotations in one of two models depending on version:
-- v2 (2024.07+): self.ui.annotation.annotations; entries carry
-- text = highlighted text, note = user note, drawer/color style,
-- pos0/pos1 = xpointer (CRE) or {page=N} table (pdf), page = xpointer
-- (CRE) or number (pdf). Sidecar key "annotations".
-- v1 (older): self.ui.bookmark.bookmarks; entries carry text = note
-- label / bookmark title, notes = highlighted text, pos0/pos1 strings.
-- Returns "v2"/"v1"/nil and the entries array.
function Bookhoard:getAnnotationStore()
if self.ui.annotation and type(self.ui.annotation.annotations) == "table" then
return "v2", self.ui.annotation.annotations
end
if self.ui.bookmark and type(self.ui.bookmark.bookmarks) == "table" then
return "v1", self.ui.bookmark.bookmarks
end
return nil, nil
end
function Bookhoard:collectAnnotations()
local bookmarks = json_util.InitArray({})
local highlights = json_util.InitArray({})
local notes = json_util.InitArray({})
local model, entries = self:getAnnotationStore()
if not model then
return bookmarks, highlights, notes
end
local file_sha256 = self:getFileSHA256()
local total_pages = self.ui.document:getPageCount()
local has_pages = self.ui.document.info.has_pages
local function wirePos(p, fallback_page)
-- v2 pdf positions are tables {page=N,x,y}; the wire format is a
-- string — the page number is the durable locator there.
if type(p) == "table" then
return tostring(p.page or fallback_page or "")
end
return p or ""
end
local function echoColor(bm)
-- Color the device can vouch for as user intent:
-- - entries we applied ourselves carry the device DEFAULT color
-- (set at apply time); pushing it back would clobber the web
-- color. Suppress it unless the user has edited the entry since
-- (datetime_updated — set by KOReader on any modification).
-- - device-native entries carry a user-chosen (or default) color;
-- those push their color as before.
if bm.bookhoard_dedup_key and not bm.datetime_updated then
return nil
end
return bm.color
end
for _, bm in ipairs(entries) do
local sel_text, user_note
if model == "v2" then
if bm.drawer then
sel_text = bm.text or "" -- highlighted text
user_note = bm.note or "" -- user note
else
-- Page bookmark. `drawer` is the only reliable highlight
-- discriminator: KOReader auto-fills text = "in Chapter X"
-- on bookmarks (updateItemByXPointer), so text-presence
-- would misclassify every bookmark as a highlight on echo.
-- The user label lives in `note`.
sel_text = ""
user_note = bm.note or ""
end
else
sel_text = bm.notes or "" -- v1: notes held the highlighted text
user_note = bm.text or "" -- v1: text held the note/label
end
-- Thin-client policy: no per-annotation CRE lookups here (a
-- getPageFromXPointer call each is the most expensive thing this
-- loop can do on weak hardware). Percentages are arithmetic for
-- paging documents and simply omitted for CRE documents — the
-- server derives them from the locator against the actual book.
local page_num = tonumber(bm.page) or 0
local percentage = has_pages and total_pages > 0 and (page_num / total_pages) or nil
local chapter = bm.chapter or ""
if type(chapter) == "table" then chapter = "" end
local entry = {
chapter = chapter,
datetime = bm.datetime or "",
notes = user_note,
pos0 = wirePos(bm.pos0, bm.page),
pos1 = wirePos(bm.pos1, bm.page),
page = tostring(bm.page or ""),
text = sel_text,
book_sha256 = file_sha256,
}
if percentage then
entry.percentage = Math.roundPercent(percentage)
end
if bm.bookhoard_dedup_key then
-- Echo identity for entries received from the server: lets the
-- server match this push to the original row instead of
-- minting a duplicate (device locators ≠ web locators, so the
-- computed key would never match).
entry.dedup_key = bm.bookhoard_dedup_key
end
local has_text = sel_text ~= ""
local has_notes = user_note ~= ""
if has_text and has_notes then
entry.type = "note"
table.insert(notes, entry)
elseif has_text then
entry.type = "highlight"
local ec = echoColor(bm)
if ec then
entry.color = ec
end
table.insert(highlights, entry)
else
entry.type = "bookmark"
entry.text = user_note -- bookmark label travels in `text`
table.insert(bookmarks, entry)
end
end
return bookmarks, highlights, notes
end
function Bookhoard:syncToProgress(progress, percentage)
logger.dbg("Bookhoard: sync to progress", progress, percentage)
if self.ui.document.info.has_pages then
local page = tonumber(progress)
if page then
self.ui:handleEvent(Event:new("GotoPage", page))
end
elseif progress and progress:match("^/body/") then
self.ui:handleEvent(Event:new("GotoXPointer", progress))
elseif percentage then
-- Navigation goes through events (ReaderRolling/ReaderPaging both
-- implement onGotoPercent and refresh the view themselves). Calling
-- document:gotoPercent directly crashed KOReader — no such method
-- exists on documents.
self.ui:handleEvent(Event:new("GotoPercent", percentage * 100))
end
end
function Bookhoard:updateProgress(ensure_networking, interactive)
if not self:isConfigured() then
if interactive then
UIManager:show(InfoMessage:new{
text = _("Please configure and register your device first."),
timeout = 3,
})
end
return
end
if not self.settings.sync_progress then return end
if not self.ui.document then
if interactive then
UIManager:show(InfoMessage:new{
text = _("No document open."),
timeout = 3,
})
end
return
end
local now = UIManager:getElapsedTimeSinceBoot()
if not interactive and now - self.push_timestamp <= API_CALL_DEBOUNCE_DELAY then
return
end
if ensure_networking
and NetworkMgr:willRerunWhenOnline(function() self:updateProgress(ensure_networking, interactive) end) then
return
end
UIManager:scheduleIn(0.5, function()
self:_doUpdateProgress(interactive)
end)
self.push_timestamp = now
end
function Bookhoard:_doUpdateProgress(interactive)
local book_data = self:collectBookData()
if not book_data then return end
if self.settings.sync_bookmarks or self.settings.sync_highlights or self.settings.sync_notes then
local bm, hl, nt = self:collectAnnotations()
book_data.bookmarks = self.settings.sync_bookmarks and bm or {}
book_data.highlights = self.settings.sync_highlights and hl or {}
book_data.notes = self.settings.sync_notes and nt or {}
else
book_data.bookmarks = {}
book_data.highlights = {}
book_data.notes = {}
end
local api = self:getAPI()
local ok, result = api:syncProgress(book_data, self.settings.sync_mode)
UIManager:nextTick(function()
if ok then
logger.dbg("Bookhoard: progress pushed successfully")
if result and result.book_results then
for _, br in ipairs(result.book_results) do
if br.synced and br.book_uuid and br.sha256 == book_data.sha256 then
self.ui.doc_settings:saveSetting("bookhoard_uuid", br.book_uuid)
self.ui.doc_settings:flush()
break
end
end
end
if interactive then
UIManager:show(InfoMessage:new{
text = _("Progress has been pushed."),
timeout = 3,
})
end
else
logger.warn("Bookhoard: failed to push progress")
if interactive then
UIManager:show(InfoMessage:new{
text = _("Failed to push progress. Check your network connection."),
timeout = 3,
})
end
end
end)
end
function Bookhoard:getProgress(ensure_networking, interactive)
if not self:isConfigured() then
if interactive then
UIManager:show(InfoMessage:new{
text = _("Please configure and register your device first."),
timeout = 3,
})
end
return
end
if not self.settings.sync_progress then return end
local now = UIManager:getElapsedTimeSinceBoot()
if not interactive and now - self.pull_timestamp <= API_CALL_DEBOUNCE_DELAY then
return
end
if ensure_networking
and NetworkMgr:willRerunWhenOnline(function() self:getProgress(ensure_networking, interactive) end) then
return
end
local book_uuid = self:getBookhoardUUID()
if not book_uuid then
-- No cached UUID yet (first open of a freshly downloaded book). Bootstrap
-- it by pushing once: the server resolves the book by SHA-256 and returns
-- the UUID, which the push handler caches. Then retry the pull.
if interactive then
UIManager:show(InfoMessage:new{
text = _("Linking this book to Bookhoard first…"),
timeout = 3,
})
end
self:_bootstrapUUIDThenPull(interactive)
return
end
UIManager:scheduleIn(0.5, function()
self:_doGetProgress(interactive)
end)
self.pull_timestamp = now
end
-- Establish the bookhoard UUID for the current document by pushing once, then
-- (once cached) perform the originally-requested pull. Used when a pull is
-- requested before any push has run on a newly downloaded book.
function Bookhoard:_bootstrapUUIDThenPull(interactive)
self:updateProgress(false, false)
UIManager:scheduleIn(2, function()
if self:getBookhoardUUID() then
self:_doGetProgress(interactive)
elseif interactive then
UIManager:show(InfoMessage:new{
text = _("Could not link this book. Check your network and try again."),
timeout = 3,
})
end
end)
end
function Bookhoard:_doGetProgress(interactive)
local book_uuid = self:getBookhoardUUID()
if not book_uuid then return end
local api = self:getAPI()
local ok, result = api:getMetadata(book_uuid)
UIManager:nextTick(function()
if not ok or not result then
if interactive then
UIManager:show(InfoMessage:new{
text = _("Failed to pull progress."),
timeout = 3,
})
end
return
end
if not result.progress then
if interactive then
UIManager:show(InfoMessage:new{
text = _("No progress found for this document."),
timeout = 3,
})
end
return
end
if result.annotations then
self:applyServerAnnotations(result.annotations)
end
local progress = result.progress
local percentage = self:getLastPercent()
local server_percentage = progress.percentage or 0
if percentage == server_percentage then
if interactive then
UIManager:show(InfoMessage:new{
text = _("Progress is already synchronized."),
timeout = 3,
})
end
return
end
local self_older = server_percentage > percentage
local nav_target
local sync_text
if self.ui.document.info.has_pages then
-- Fixed-layout: the page index is the canonical locator. CFI/xpointer
-- are meaningless for image-based content, so use progress.page directly.
local total = progress.total_pages or self.ui.document:getPageCount()
local target_page = progress.page
or math.min(Math.round(server_percentage * total), total)
nav_target = progress.page
sync_text = T(_("Sync to page %1 of %2 from server?"), target_page, total)
else
nav_target = progress.koreader_xpointer or progress.epubcfi or progress.page
local total = self.ui.document:getPageCount()
local target_page = math.min(Math.round(server_percentage * total), total)
sync_text = T(_("Sync to page %1 of %2 from server?"), target_page, total)
end
if self_older then
if self.settings.sync_forward == SYNC_STRATEGY.SILENT then
self:syncToProgress(nav_target, server_percentage)
self:_showSyncedMessage()
elseif self.settings.sync_forward == SYNC_STRATEGY.PROMPT then
UIManager:show(ConfirmBox:new{
text = sync_text,
ok_callback = function()
self:syncToProgress(nav_target, server_percentage)
end,
})
end
else
if self.settings.sync_backward == SYNC_STRATEGY.SILENT then
self:syncToProgress(nav_target, server_percentage)
self:_showSyncedMessage()
elseif self.settings.sync_backward == SYNC_STRATEGY.PROMPT then
UIManager:show(ConfirmBox:new{
text = sync_text,
ok_callback = function()
self:syncToProgress(nav_target, server_percentage)
end,
})
end
end
end)
end
function Bookhoard:applyServerAnnotations(annotations)
if not annotations then return end
local model, entries = self:getAnnotationStore()
if not model then return end
local changed = false
local has_pages = self.ui.document.info.has_pages
local function findLocal(server_entry)
-- Identity match by dedup key: survives pos0 drift (improved
-- server conversion) and, crucially, never cross-matches a
-- DIFFERENT annotation that merely shares the position.
local key = server_entry.dedup_key
if key and key ~= "" then
for i, bm in ipairs(entries) do
if bm.bookhoard_dedup_key == key then
return i
end
end
return nil -- keyed but not present: a real new entry
end
-- Legacy serve (no key): fall back to pos0 matching.
local pos0 = server_entry.pos0 or ""
if pos0 == "" then return nil end
for i, bm in ipairs(entries) do
local p = bm.pos0
if type(p) == "table" then p = tostring(p.page or "") end
if p == pos0 then
return i
end
end
return nil
end
-- Get/set the highlighted text and the user note with model-appropriate
-- field names (v2: text/note; v1: notes/text).
local function getFields(bm)
if model == "v2" then
return bm.text or "", bm.note or ""
end
return bm.notes or "", bm.text or ""
end
local function setFields(bm, sel_text, user_note)
if model == "v2" then
bm.text = sel_text ~= "" and sel_text or nil
bm.note = user_note ~= "" and user_note or nil
else
bm.notes = sel_text
bm.text = user_note
end
end
local function isValidPos0(pos0)
-- Rolling (CRE) documents address bookmarks/highlights by xpointer;
-- paging documents (PDF/comics/DjVu) by page number. Anything else
-- (a raw "cfi:" locator, an epubcfi(...) string, a JSON anchor, …)
-- cannot be placed in this document and must not become a local
-- bookmark: it could never be matched again and would be re-pushed
-- to the server as a junk duplicate on the next sync.
if pos0 == "" then return false end
if has_pages then
return tonumber(pos0) ~= nil
end
return pos0:sub(1, 6) == "/body/"
end
local function makePageFromPos(pos0)
if has_pages then
local page = tonumber(pos0:match("(%d+)$"))
return page or pos0
end
return pos0
end
-- v2 pdf positions are {page=N} tables; CRE keeps xpointer strings.
local function makeLocalPos(pos)
if has_pages then
local page = tonumber(pos)
return page and { page = page } or nil
end
return pos
end
local function addOrUpdate(server_entry, has_text)
local pos0 = server_entry.pos0 or ""
if not isValidPos0(pos0) then return end
local srv_text = server_entry.text or ""
local srv_notes = server_entry.notes or ""
local idx = findLocal(server_entry)
if idx then
local bm = entries[idx]
local cur_text, cur_note = getFields(bm)
if cur_text ~= srv_text or cur_note ~= srv_notes then
setFields(bm, srv_text, srv_notes)
if server_entry.color and server_entry.color ~= "" then bm.color = server_entry.color end
changed = true
end
elseif model == "v2" then
-- `drawer` marks a renderable highlight in v2; its absence
-- makes the entry a page bookmark (label goes in `note`).
local entry = {
datetime = server_entry.datetime ~= "" and server_entry.datetime or nil,
chapter = server_entry.chapter ~= "" and server_entry.chapter or nil,
pos0 = makeLocalPos(pos0),
pos1 = makeLocalPos(server_entry.pos1 ~= "" and server_entry.pos1 or pos0),
page = makePageFromPos(pos0),
}
if server_entry.dedup_key and server_entry.dedup_key ~= "" then
entry.bookhoard_dedup_key = server_entry.dedup_key
end
if has_text then
entry.drawer = "lighten"
entry.text = srv_text
if srv_notes ~= "" then entry.note = srv_notes end
-- Match the device's native highlight look exactly: native
-- highlights carry view.highlight.saved_color ("yellow" on
-- color screens); a nil color renders grey (darkenRect).
-- Synced entries still never push this color back (see
-- collectAnnotations) — the web color only changes when the
-- highlight is edited here.
local default_color = (self.ui.view and self.ui.view.highlight
and self.ui.view.highlight.saved_color) or "yellow"
entry.color = default_color
else
if srv_text ~= "" then entry.note = srv_text end -- bookmark label
end
self.ui.annotation:addItem(entry)
changed = true
else
local entry = {
page = makePageFromPos(pos0),
pos0 = pos0,
pos1 = server_entry.pos1 or pos0,
datetime = server_entry.datetime or "",
notes = srv_text,
text = srv_notes,
}
if server_entry.dedup_key and server_entry.dedup_key ~= "" then
entry.bookhoard_dedup_key = server_entry.dedup_key
end
if server_entry.chapter and server_entry.chapter ~= "" then entry.chapter = server_entry.chapter end
table.insert(entries, entry)
changed = true
end
end
if self.settings.sync_highlights and annotations.highlights then
for _, hl in ipairs(annotations.highlights) do
addOrUpdate(hl, true)
end
end
if self.settings.sync_notes and annotations.notes then
for _, note in ipairs(annotations.notes) do
addOrUpdate(note, true)
end
end
if self.settings.sync_bookmarks and annotations.bookmarks then
for _, bm in ipairs(annotations.bookmarks) do
addOrUpdate(bm, false)
end
end
local function findLocalByTombstone(del, want_highlight)
local key = del.dedup_key
if key and key ~= "" then
for i, bm in ipairs(entries) do
if bm.bookhoard_dedup_key == key then
return i
end
end
return nil -- keyed tombstone: only its own entry may be removed
end
-- Legacy tombstone (no key): pos0 match, restricted to the same
-- annotation kind so a highlight tombstone never eats a bookmark
-- that shares the position.
local pos0 = del.pos0 or ""
if pos0 == "" then return nil end
for i, bm in ipairs(entries) do
local p = bm.pos0
if type(p) == "table" then p = tostring(p.page or "") end
if p == pos0 and (not not bm.drawer) == want_highlight then
return i
end
end
return nil
end
if annotations.deleted_highlights then
for _, del in ipairs(annotations.deleted_highlights) do
local idx = findLocalByTombstone(del, true)
local bm = idx and entries[idx]
local cur_text = bm and (getFields(bm)) or ""
if idx and cur_text ~= "" then
table.remove(entries, idx)
changed = true
end
end
end
if annotations.deleted_bookmarks then
for _, del in ipairs(annotations.deleted_bookmarks) do
local idx = findLocalByTombstone(del, false)
if idx then
table.remove(entries, idx)
changed = true
end
end
end
if changed then
-- addItem() inserts v2 entries at their sorted position already;
-- explicit re-sorting is a v1-model need only.
if model == "v1" and self.ui.bookmark and self.ui.bookmark.onSortBookmarks then
self.ui.bookmark:onSortBookmarks()
end
if self.ui.saveSettings then
self.ui:saveSettings()
end
-- ReaderView caches each rendered page's highlight boxes and only
-- invalidates them on AnnotationsModified — without this, applied
-- annotations don't paint until restart despite the repaint.
-- A non-table payload clears the whole cache (per-item payloads
-- only remove the pages of the listed items).
self.ui:handleEvent(Event:new("AnnotationsModified", nil))
if self.ui.dialog then
UIManager:setDirty(self.ui.dialog, "ui")
end
logger.dbg("Bookhoard: annotations updated from server")
end
end
function Bookhoard:_showSyncedMessage()
UIManager:show(InfoMessage:new{
text = _("Progress has been synchronized."),
timeout = 3,
})
end
function Bookhoard:_onCloseDocument()
self.onResume = nil
self.onSuspend = nil
NetworkMgr:goOnlineToRun(function()
self:updateProgress(false, false)
end)
end
function Bookhoard:schedulePeriodicPush()
UIManager:unschedule(self.periodic_push_task)
UIManager:scheduleIn(PERIODIC_PUSH_DELAY, self.periodic_push_task)
self.periodic_push_scheduled = true
end
function Bookhoard:_onPageUpdate(page)
if page == nil then return end
if self.last_page ~= page then
self.last_page = page
self.page_update_counter = self.page_update_counter + 1
if self.periodic_push_scheduled
or (self.settings.pages_before_update
and self.page_update_counter >= self.settings.pages_before_update) then
self:schedulePeriodicPush()
end
end
end
function Bookhoard:_onResume()
if Device:hasWifiRestore() and NetworkMgr.wifi_was_on
and G_reader_settings:isTrue("auto_restore_wifi") then
return
end
UIManager:scheduleIn(1, function()
self:getProgress(true, false)
end)
end
function Bookhoard:_onSuspend()
self:updateProgress(true, false)
end
function Bookhoard:_onNetworkConnected()
UIManager:scheduleIn(0.5, function()
self:getProgress(false, false)
end)
end
function Bookhoard:_onNetworkDisconnecting()
self:updateProgress(false, false)
end
function Bookhoard:onCloseWidget()
UIManager:unschedule(self.periodic_push_task)
self.periodic_push_task = nil
end
return Bookhoard