bookhoard.koplugin/main.lua

local ConfirmBox = require("ui/widget/confirmbox")
local DataStorage = require("datastorage")
local Device = require("device")
local Event = require("ui/event")
local InfoMessage = require("ui/widget/infomessage")
local InputDialog = require("ui/widget/inputdialog")
local LuaSettings = require("luasettings")
local Math = require("optmath")
local NetworkMgr = require("ui/network/manager")
local SpinWidget = require("ui/widget/spinwidget")
local UIManager = require("ui/uimanager")
local WidgetContainer = require("ui/widget/container/widgetcontainer")
local json = require("json")
local json_util = require("json.util")
local logger = require("logger")
local sha2 = require("ffi/sha2")
local time = require("ui/time")
local util = require("util")
local lfs = require("libs/libkoreader-lfs")
local T = require("ffi/util").template
local _ = require("gettext")

local BookhoardAPI = require("BookhoardAPI")

local SYNC_STRATEGY = {
    PROMPT = 1,
    SILENT = 2,
    DISABLE = 3,
}

local SYNC_MODE = {
    IMMEDIATE = "immediate",
    CHECKPOINT = "checkpoint",
}

local API_CALL_DEBOUNCE_DELAY = time.s(25)
local PERIODIC_PUSH_DELAY = 10

local sha256hex = sha2.sha256hex or sha2.sha256

local Bookhoard = WidgetContainer:extend({
    name = "bookhoard",
    is_doc_only = false,
    title = _("Bookhoard Server"),
    settings_key = "bookhoard",

    push_timestamp = nil,
    pull_timestamp = nil,
    page_update_counter = nil,
    last_page = nil,
    periodic_push_task = nil,
    periodic_push_scheduled = nil,
    registration_poll_scheduled = nil,

    settings = nil,
})

Bookhoard.default_settings = {
    server_url = nil,
    auth_token = nil,
    device_id = nil,
    auto_sync = false,
    pages_before_update = 50,
    sync_forward = SYNC_STRATEGY.PROMPT,
    sync_backward = SYNC_STRATEGY.DISABLE,
    sync_progress = true,
    sync_bookmarks = true,
    sync_highlights = true,
    sync_notes = true,
    sync_mode = SYNC_MODE.IMMEDIATE,
}

function Bookhoard:init()
    self.push_timestamp = 0
    self.pull_timestamp = 0
    self.page_update_counter = 0
    self.last_page = -1
    self.periodic_push_scheduled = false
    self.registration_poll_scheduled = false

    self.periodic_push_task = function()
        self.periodic_push_scheduled = false
        self.page_update_counter = 0
        self:updateProgress(false, false)
    end

    -- readSetting returns the STORED table verbatim when one exists — it
    -- does not merge defaults. A settings file saved by an older plugin
    -- version lacks the annotation toggles entirely, so sync_bookmarks/
    -- highlights/notes read as nil, the "or" in _doUpdateProgress fell
    -- through to the else branch, and every push carried EMPTY annotation
    -- arrays while progress synced fine. Merge missing defaults (and
    -- persist once so the file and the menu checkboxes self-heal).
    self.settings = G_reader_settings:readSetting(self.settings_key, {})
    if type(self.settings) ~= "table" then
        self.settings = {}
    end
    local settings_merged = false
    for key, value in pairs(self.default_settings) do
        if self.settings[key] == nil then
            self.settings[key] = value
            settings_merged = true
        end
    end
    if settings_merged then
        G_reader_settings:saveSetting(self.settings_key, self.settings)
        logger.info("Bookhoard: merged new default settings (older settings file)")
    end

    if self.settings.auto_sync
        and Device:hasSeamlessWifiToggle()
        and G_reader_settings:readSetting("wifi_enable_action") ~= "turn_on" then
        self.settings.auto_sync = false
        logger.warn("Bookhoard: auto-sync disabled because wifi_enable_action is not turn_on")
    end

    self.ui.menu:registerToMainMenu(self)
    self:setupMenuOrder()
end

function Bookhoard:onReaderReady()
    if self.settings.auto_sync then
        UIManager:nextTick(function()
            -- Pull-first on every open. A freshly downloaded book has no cached
            -- bookhoard UUID yet; getProgress resolves it by SHA-256 with a
            -- read-only lookup (no progress push). The previous push-to-
            -- bootstrap strategy transmitted the device's first-page position
            -- to the server on books already mid-read elsewhere, creating a
            -- sync conflict before the first pull could ever happen.
            self:getProgress(true, false)
        end)
    end
    self:registerEvents()
    self.last_page = self.ui:getCurrentPage()
end

function Bookhoard:registerEvents()
    if self.settings.auto_sync then
        self.onCloseDocument = self._onCloseDocument
        self.onPageUpdate = self._onPageUpdate
        self.onResume = self._onResume
        self.onSuspend = self._onSuspend
        self.onNetworkConnected = self._onNetworkConnected
        self.onNetworkDisconnecting = self._onNetworkDisconnecting
    else
        self.onCloseDocument = nil
        self.onPageUpdate = nil
        self.onResume = nil
        self.onSuspend = nil
        self.onNetworkConnected = nil
        self.onNetworkDisconnecting = nil
    end
end

function Bookhoard:getAPI()
    return BookhoardAPI:new({
        server_url = self.settings.server_url,
        auth_token = self.settings.auth_token,
    })
end

function Bookhoard:isConfigured()
    return self.settings.server_url and self.settings.auth_token
end

function Bookhoard:getSyncPeriod()
    if not self.settings.auto_sync then
        return _("Not available")
    end
    local period = self.settings.pages_before_update
    if period and period > 0 then
        return period
    end
    return _("Never")
end

function Bookhoard:addToMainMenu(menu_items)
    menu_items.bookhoard_sync = {
        text = _("Bookhoard sync"),
        sorting_hint = "tools",
        sub_item_table = self:buildMainMenu(),
    }
end

function Bookhoard:buildMainMenu()
    local items = {}

    if self:isConfigured() then
        table.insert(items, {
            text = _("Push progress from this device"),
            callback = function()
                self:updateProgress(true, true)
            end,
        })
        table.insert(items, {
            text = _("Pull progress from server"),
            callback = function()
                self:getProgress(true, true)
            end,
            separator = true,
        })
    end

    table.insert(items, {
        text = _("Server URL"),
        keep_menu_open = true,
        tap_input_func = function()
            return {
                title = _("Bookhoard server URL"),
                input = self.settings.server_url or "http://",
                callback = function(input)
                    self.settings.server_url = input ~= "" and input or nil
                    G_reader_settings:saveSetting(self.settings_key, self.settings)
                end,
            }
        end,
    })

    if self:isConfigured() then
        table.insert(items, {
            text = _("Device info"),
            keep_menu_open = true,
            callback = function()
                UIManager:show(InfoMessage:new{
                    text = T(_("Device ID: %1\nServer: %2"),
                        self.settings.device_id or _("unknown"),
                        self.settings.server_url),
                })
            end,
        })
        table.insert(items, {
            text = _("Disconnect"),
            keep_menu_open = true,
            callback = function()
                UIManager:show(ConfirmBox:new{
                    text = _("Disconnect from Bookhoard server?"),
                    ok_text = _("Disconnect"),
                    ok_callback = function()
                        self.settings.auth_token = nil
                        self.settings.device_id = nil
                        self.settings.auto_sync = false
                        G_reader_settings:saveSetting(self.settings_key, self.settings)
                        self:registerEvents()
                        UIManager:askForRestart()
                    end,
                })
            end,
            separator = true,
        })
    else
        table.insert(items, {
            text = _("Register device"),
            keep_menu_open = true,
            callback = function()
                self:startRegistration()
            end,
            separator = true,
        })
    end

    table.insert(items, {
        text = _("Automatically push progress"),
        checked_func = function() return self.settings.auto_sync end,
        help_text = _([[This may lead to prompts about toggling WiFi on document close and suspend/resume, depending on your device's connectivity.]]),
        callback = function()
            self:toggleAutoSync()
        end,
    })

    table.insert(items, {
        text_func = function()
            return T(_("Periodically sync every # pages (%1)"), self:getSyncPeriod())
        end,
        enabled_func = function() return self.settings.auto_sync end,
        keep_menu_open = true,
        callback = function(touchmenu_instance)
            local spin = SpinWidget:new{
                text = _([[Number of page turns between progress updates. Set to 0 to disable.]]),
                value = self.settings.pages_before_update or 0,
                value_min = 0,
                value_max = 999,
                value_step = 1,
                value_hold_step = 10,
                ok_text = _("Set"),
                title_text = _("Pages before update"),
                default_value = 50,
                callback = function(spin)
                    self.settings.pages_before_update = spin.value > 0 and spin.value or nil
                    G_reader_settings:saveSetting(self.settings_key, self.settings)
                    if touchmenu_instance then touchmenu_instance:updateItems() end
                end,
            }
            UIManager:show(spin)
        end,
    })

    table.insert(items, {
        text_func = function()
            return T(_("Sync mode (%1)"),
                self.settings.sync_mode == SYNC_MODE.IMMEDIATE and _("immediate") or _("checkpoint"))
        end,
        sub_item_table = {
            {
                text = _("Immediate"),
                checked_func = function()
                    return self.settings.sync_mode == SYNC_MODE.IMMEDIATE
                end,
                callback = function()
                    self.settings.sync_mode = SYNC_MODE.IMMEDIATE
                    G_reader_settings:saveSetting(self.settings_key, self.settings)
                end,
            },
            {
                text = _("Checkpoint"),
                checked_func = function()
                    return self.settings.sync_mode == SYNC_MODE.CHECKPOINT
                end,
                callback = function()
                    self.settings.sync_mode = SYNC_MODE.CHECKPOINT
                    G_reader_settings:saveSetting(self.settings_key, self.settings)
                end,
            },
        },
        separator = true,
    })

    table.insert(items, {
        text = _("Sync behavior"),
        sub_item_table = {
            {
                text_func = function()
                    return T(_("Sync to a newer state (%1)"),
                        self:getStrategyName(self.settings.sync_forward))
                end,
                sub_item_table = {
                    {
                        text = _("Silently"),
                        checked_func = function()
                            return self.settings.sync_forward == SYNC_STRATEGY.SILENT
                        end,
                        callback = function()
                            self.settings.sync_forward = SYNC_STRATEGY.SILENT
                            G_reader_settings:saveSetting(self.settings_key, self.settings)
                        end,
                    },
                    {
                        text = _("Prompt"),
                        checked_func = function()
                            return self.settings.sync_forward == SYNC_STRATEGY.PROMPT
                        end,
                        callback = function()
                            self.settings.sync_forward = SYNC_STRATEGY.PROMPT
                            G_reader_settings:saveSetting(self.settings_key, self.settings)
                        end,
                    },
                    {
                        text = _("Never"),
                        checked_func = function()
                            return self.settings.sync_forward == SYNC_STRATEGY.DISABLE
                        end,
                        callback = function()
                            self.settings.sync_forward = SYNC_STRATEGY.DISABLE
                            G_reader_settings:saveSetting(self.settings_key, self.settings)
                        end,
                    },
                },
            },
            {
                text_func = function()
                    return T(_("Sync to an older state (%1)"),
                        self:getStrategyName(self.settings.sync_backward))
                end,
                sub_item_table = {
                    {
                        text = _("Silently"),
                        checked_func = function()
                            return self.settings.sync_backward == SYNC_STRATEGY.SILENT
                        end,
                        callback = function()
                            self.settings.sync_backward = SYNC_STRATEGY.SILENT
                            G_reader_settings:saveSetting(self.settings_key, self.settings)
                        end,
                    },
                    {
                        text = _("Prompt"),
                        checked_func = function()
                            return self.settings.sync_backward == SYNC_STRATEGY.PROMPT
                        end,
                        callback = function()
                            self.settings.sync_backward = SYNC_STRATEGY.PROMPT
                            G_reader_settings:saveSetting(self.settings_key, self.settings)
                        end,
                    },
                    {
                        text = _("Never"),
                        checked_func = function()
                            return self.settings.sync_backward == SYNC_STRATEGY.DISABLE
                        end,
                        callback = function()
                            self.settings.sync_backward = SYNC_STRATEGY.DISABLE
                            G_reader_settings:saveSetting(self.settings_key, self.settings)
                        end,
                    },
                },
            },
        },
        separator = true,
    })

    table.insert(items, {
        text = _("What to sync"),
        sub_item_table = {
            {
                text = _("Reading progress"),
                checked_func = function() return self.settings.sync_progress end,
                callback = function()
                    self.settings.sync_progress = not self.settings.sync_progress
                    G_reader_settings:saveSetting(self.settings_key, self.settings)
                end,
            },
            {
                text = _("Bookmarks"),
                checked_func = function() return self.settings.sync_bookmarks end,
                callback = function()
                    self.settings.sync_bookmarks = not self.settings.sync_bookmarks
                    G_reader_settings:saveSetting(self.settings_key, self.settings)
                end,
            },
            {
                text = _("Highlights"),
                checked_func = function() return self.settings.sync_highlights end,
                callback = function()
                    self.settings.sync_highlights = not self.settings.sync_highlights
                    G_reader_settings:saveSetting(self.settings_key, self.settings)
                end,
            },
            {
                text = _("Notes"),
                checked_func = function() return self.settings.sync_notes end,
                callback = function()
                    self.settings.sync_notes = not self.settings.sync_notes
                    G_reader_settings:saveSetting(self.settings_key, self.settings)
                end,
            },
        },
        separator = true,
    })

    table.insert(items, {
        text = _("Setup OPDS catalog"),
        keep_menu_open = true,
        enabled_func = function() return self:isConfigured() end,
        callback = function()
            if self:setupOPDS() then
                UIManager:show(InfoMessage:new{
                    text = _("Bookhoard OPDS catalog added! Find it in Home → OPDS Catalog."),
                    timeout = 3,
                })
            else
                UIManager:show(InfoMessage:new{
                    text = _("Please configure and register your device first."),
                    timeout = 3,
                })
            end
        end,
    })

    return items
end

function Bookhoard:getStrategyName(strategy)
    if strategy == SYNC_STRATEGY.PROMPT then
        return _("Prompt")
    elseif strategy == SYNC_STRATEGY.SILENT then
        return _("Auto")
    else
        return _("Disable")
    end
end

function Bookhoard:toggleAutoSync()
    if not self.settings.auto_sync
        and Device:hasSeamlessWifiToggle()
        and G_reader_settings:readSetting("wifi_enable_action") ~= "turn_on" then
        UIManager:show(InfoMessage:new{
            text = _("Set 'Action when Wi-Fi is off' to 'turn on' in Network settings to enable auto sync."),
        })
        return
    end
    self.settings.auto_sync = not self.settings.auto_sync
    self:registerEvents()
    G_reader_settings:saveSetting(self.settings_key, self.settings)

    if self.settings.auto_sync and self.ui.doc_settings then
        self:getProgress(true, true)
    end
end

function Bookhoard:setupMenuOrder()
    local settings_dir = DataStorage:getSettingsDir()
    self:patchMenuOrderFile(settings_dir .. "/reader_menu_order.lua")
    self:patchMenuOrderFile(settings_dir .. "/filemanager_menu_order.lua")
end

function Bookhoard:patchMenuOrderFile(filepath)
    local default_tools = {
        "read_timer",
        "calibre",
        "bookhoard_sync",
        "exporter",
        "statistics",
        "progress_sync",
        "move_to_archive",
        "wallabag",
        "news_downloader",
        "text_editor",
        "profiles",
        "qrclipboard",
        "----------------------------",
        "more_tools",
    }

    local attr = lfs.attributes(filepath)
    if not attr then
        local f = io.open(filepath, "w")
        if not f then return end
        f:write("return {\n    tools = {\n")
        for _, id in ipairs(default_tools) do
            f:write('        "' .. id .. '",\n')
        end
        f:write("    },\n}\n")
        f:close()
        logger.dbg("Bookhoard: created menu order file", filepath)
        return
    end

    local existing = dofile(filepath)
    if not existing or type(existing) ~= "table" then return end

    local tools = existing.tools
    if not tools or type(tools) ~= "table" then return end

    for _, id in ipairs(tools) do
        if id == "bookhoard" then return end
    end

    local insert_pos = nil
    for i, id in ipairs(tools) do
        if id == "calibre" then
            insert_pos = i + 1
            break
        end
    end
    if not insert_pos then
        insert_pos = 2
    end

    table.insert(tools, insert_pos, "bookhoard")
    existing.tools = tools

    local f = io.open(filepath, "w")
    if not f then return end
    f:write("return {\n")
    for key, val in pairs(existing) do
        if type(val) == "table" then
            f:write("    " .. key .. " = {\n")
            for _, id in ipairs(val) do
                f:write('        "' .. id .. '",\n')
            end
            f:write("    },\n")
        end
    end
    f:write("}\n")
    f:close()
    logger.dbg("Bookhoard: patched menu order file", filepath)
end

function Bookhoard:startRegistration()
    if not self.settings.server_url or self.settings.server_url == "" then
        UIManager:show(InfoMessage:new{
            text = _("Please set your server URL first."),
            timeout = 3,
        })
        return
    end

    if NetworkMgr:willRerunWhenOnline(function() self:startRegistration() end) then
        return
    end

    local device_name = Device.model or "KOReader Device"
    local device_identifier = Device:info() or device_name

    UIManager:show(InfoMessage:new{
        text = _("Registering device…"),
        timeout = 1,
    })

    UIManager:scheduleIn(0.5, function()
        local api = BookhoardAPI:new({ server_url = self.settings.server_url })
        local ok, result = api:registerDevice(device_name, device_identifier)

        if not ok then
            UIManager:show(InfoMessage:new{
                text = T(_("Registration failed: %1"),
                    result and result.error or _("unknown error")),
            })
            return
        end

        self.registration_id = result.registration_id

        self.waiting_dialog = InfoMessage:new{
            text = T(_("Device registered on server.\n\nOpen your Bookhoard web UI and go to:\n%1/devices\n\nApprove this device in the \"Pending Device Registrations\" section.\n\nWaiting for approval…"), self.settings.server_url),
        }
        UIManager:show(self.waiting_dialog)

        self:startRegistrationPoll()
    end)
end

function Bookhoard:startRegistrationPoll()
    if self.registration_poll_scheduled then return end
    self.registration_poll_scheduled = true

    local function poll()
        self.registration_poll_scheduled = false

        if not self.registration_id then return end

        local api = BookhoardAPI:new({ server_url = self.settings.server_url })
        local ok, result = api:checkRegistrationStatus(self.registration_id)

        if not ok then
            if result and result.status == 410 then
                self.registration_id = nil
                UIManager:show(InfoMessage:new{
                    text = _("Registration expired. Please try again."),
                })
                return
            end
            self.registration_poll_scheduled = true
            UIManager:scheduleIn(3, poll)
            return
        end

        if result.status == "approved" then
            self.registration_id = nil
            self.settings.auth_token = result.auth_token
            self.settings.device_id = result.device_id and tostring(result.device_id) or nil
            G_reader_settings:saveSetting(self.settings_key, self.settings)
            self:registerEvents()
            self:setupOPDS()
            if self.waiting_dialog then
                UIManager:close(self.waiting_dialog)
                self.waiting_dialog = nil
            end
            UIManager:show(InfoMessage:new{
                text = _("Device registered successfully!"),
                timeout = 3,
            })
        elseif result.status == "pending" then
            self.registration_poll_scheduled = true
            UIManager:scheduleIn(3, poll)
        else
            self.registration_id = nil
            UIManager:show(InfoMessage:new{
                text = T(_("Registration %1"), result.status or _("failed")),
            })
        end
    end

    UIManager:scheduleIn(3, poll)
end

function Bookhoard:setupOPDS()
    if not self.settings.server_url or not self.settings.device_id
        or not self.settings.auth_token then
        return false
    end

    local opds_base_url = self.settings.server_url
        .. "/opds/devices/" .. self.settings.device_id .. "/catalog"
    local opds_url = opds_base_url .. "?token=" .. self.settings.auth_token

    local opds_settings_file = DataStorage:getSettingsDir() .. "/opds.lua"
    local opds_settings = LuaSettings:open(opds_settings_file)
    local servers = opds_settings:readSetting("servers", {})

    local found = false
    for i, server in ipairs(servers) do
        if server.url and (
            server.url == opds_url
            or server.url == opds_base_url
            or server.url:sub(1, #opds_base_url + 1) == opds_base_url .. "?"
        ) then
            servers[i] = {
                title = "Bookhoard",
                url = opds_url,
            }
            found = true
            break
        end
    end

    if not found then
        table.insert(servers, {
            title = "Bookhoard",
            url = opds_url,
        })
    end

    opds_settings:saveSetting("servers", servers)
    opds_settings:flush()

    if self.ui.opds then
        self.ui.opds.servers = servers
    end

    return true
end

function Bookhoard:getLastPercent()
    if self.ui.document.info.has_pages then
        return Math.roundPercent(self.ui.paging:getLastPercent())
    else
        return Math.roundPercent(self.ui.rolling:getLastPercent())
    end
end

function Bookhoard:getLastProgress()
    if self.ui.document.info.has_pages then
        return self.ui.paging:getLastProgress()
    else
        return self.ui.rolling:getLastProgress()
    end
end

function Bookhoard:getFileSHA256()
    local cached = self.ui.doc_settings:readSetting("bookhoard_sha256")
    if cached then return cached end

    local file = io.open(self.ui.document.file, "rb")
    if not file then return nil end
    local data = file:read("*a")
    file:close()

    local hash = sha256hex(data)
    if hash then
        self.ui.doc_settings:saveSetting("bookhoard_sha256", hash)
    end
    return hash
end

function Bookhoard:getBookhoardUUID()
    if not self.ui.doc_settings then return nil end
    return self.ui.doc_settings:readSetting("bookhoard_uuid")
end

function Bookhoard:getContextText()
    if not self.ui.document or self.ui.document.info.has_pages then
        return ""
    end
    local xp = self:getLastProgress()
    if not xp then return "" end

    local function clean(s)
        if not s or s == "" then return "" end
        if #s > 100 then
            s = s:sub(1, 100)
        end
        return s:gsub("%s+", " "):match("^%s*(.-)%s*$") or ""
    end

    local function tryPointer(p)
        if not p or p == "" then return "" end
        local ok, text = pcall(function()
            return self.ui.document:getTextFromXPointer(p)
        end)
        if not ok then return "" end
        return text or ""
    end

    local text = tryPointer(xp)
    if text == "" then
        local ok, nxp = pcall(function()
            return self.ui.document:getNormalizedXPointer(xp)
        end)
        if ok and nxp and nxp ~= "" and nxp ~= xp then
            text = tryPointer(nxp)
        end
    end
    if text == "" then return "" end

    local cleaned = clean(text)
    -- Single text nodes split by inline markup (e.g. drop-caps like
    -- <p><span>C</span>onvergence…</p>) return just "C" here. That one
    -- character then false-positives the server text search to the first
    -- "C" in the chapter (doc start). Walk up to the enclosing block
    -- element(s) so we capture words, not the node. Kindle-light: at most
    -- 3 extra local reads, only on this tiny-text path, zero extra network.
    local function runeLen(s)
        -- Cheap UTF-8 aware length without dependencies.
        local _, n = s:gsub("[^\128-\191]", "")
        return n
    end
    if cleaned ~= "" and runeLen(cleaned) < 15 then
        local parent = xp
        for _ = 1, 3 do
            -- Strip one trailing path segment: "/text().N" or "/span[1]" etc.
            local stripped = parent:gsub("/[^/]+$", "")
            if not stripped or stripped == "" or stripped == parent then
                break
            end
            parent = stripped
            -- Don't walk past the document body; element pointers above the
            -- block would return whole-chapter text.
            if parent:match("/body%s*$") or parent:match("DocFragment%[%d+%]$") then
                break
            end
            local ptext = tryPointer(parent)
            if ptext and ptext ~= "" then
                local pcleaned = clean(ptext)
                if pcleaned ~= "" and runeLen(pcleaned) >= 15 then
                    return pcleaned
                end
                -- Keep the longest thing seen in case no level reaches 15.
                if runeLen(pcleaned) > runeLen(cleaned) then
                    cleaned = pcleaned
                end
            end
        end
        return cleaned
    end

    local char_offset = tonumber(xp:match("text%(%)%.?(%d+)")) or 0
    if char_offset > 0 and char_offset < #text then
        text = text:sub(char_offset + 1)
    end

    return clean(text)
end

function Bookhoard:collectBookData()
    local props = self.ui.doc_props
    local file_path = self.ui.document.file

    local title = props.display_title or ""
    local authors = {}
    if props.authors then
        authors = { props.authors }
    end
    json_util.InitArray(authors)

    local file_sha256 = self:getFileSHA256()
    local book_uuid = self:getBookhoardUUID()

    local percentage = self:getLastPercent()
    local progress = self:getLastProgress()
    local page = self.ui:getCurrentPage()
    local total_pages = self.ui.document:getPageCount()
    local context_text = self:getContextText()

    local book_data = {
        uuid = book_uuid,
        sha256 = file_sha256,
        title = title,
        authors = authors,
        percentage = percentage,
        context_text = context_text,
        page = page,
        total_pages = total_pages,
        file_path = file_path,
        device_info = {
            koreader_version = require("version"):getCurrentRevision(),
            device_model = Device.model,
        },
    }

    -- epubcfi (a CREngine xpointer) is only meaningful for reflowable (rolling)
    -- documents. Paging docs (PDF/comics/DjVu) carry their position in
    -- page/total_pages; a bare page number here would be a JSON number into a
    -- server *string field and fail the bind. Reflowable EPUBs are always
    -- rolling, so they keep sending the xpointer exactly as before.
    if not self.ui.document.info.has_pages then
        book_data.epubcfi = progress
    end

    return book_data
end

-- KOReader stores annotations in one of two models depending on version:
--  v2 (2024.07+): self.ui.annotation.annotations; entries carry
--     text = highlighted text, note = user note, drawer/color style,
--     pos0/pos1 = xpointer (CRE) or {page=N} table (pdf), page = xpointer
--     (CRE) or number (pdf). Sidecar key "annotations".
--  v1 (older):    self.ui.bookmark.bookmarks; entries carry text = note
--     label / bookmark title, notes = highlighted text, pos0/pos1 strings.
-- Returns "v2"/"v1"/nil and the entries array.
function Bookhoard:getAnnotationStore()
    if self.ui.annotation and type(self.ui.annotation.annotations) == "table" then
        return "v2", self.ui.annotation.annotations
    end
    if self.ui.bookmark and type(self.ui.bookmark.bookmarks) == "table" then
        return "v1", self.ui.bookmark.bookmarks
    end
    return nil, nil
end

-- ============================================================================
-- Deletion propagation
--
-- The push is upsert-only: absence from the annotation arrays must never be
-- read as a delete (a category toggled off in "What to sync" would otherwise
-- wipe the server). Instead the sidecar remembers the dedup keys of every
-- annotation the server has served us (bookhoard_known_keys). When one of
-- those keys is no longer present in the local store, the user deleted it on
-- this device, and the next push reports it via deleted_highlights /
-- deleted_bookmarks so the server tombstones it (restorable from the web
-- history). Keys are only ever learned from server pulls, so a device-native
-- annotation can never be mis-flagged.
-- ============================================================================

function Bookhoard:getKnownAnnotationKeys()
    if not self.ui.doc_settings then return nil end
    local known = self.ui.doc_settings:readSetting("bookhoard_known_keys")
    if type(known) ~= "table" then return nil end
    return known
end

-- Record one dedup key of an annotation that exists locally (applied now or
-- already present), so a later local absence can be reported as a deliberate
-- deletion. kind: "highlight" or "bookmark" (device notes are highlights-
-- with-notes server-side). Returns true when the sidecar changed.
function Bookhoard:rememberAnnotationKey(kind, key)
    if not key or key == "" or not self.ui.doc_settings then return false end
    local known = self:getKnownAnnotationKeys() or {}
    if known[key] == kind then return false end
    known[key] = kind
    self.ui.doc_settings:saveSetting("bookhoard_known_keys", known)
    return true
end

-- Drop keys the server tombstoned: the deletion already happened server-side,
-- so this device must never report it again.
function Bookhoard:forgetAnnotationKeys(tombstones)
    if not tombstones or not self.ui.doc_settings then return false end
    local known = self:getKnownAnnotationKeys()
    if not known then return false end
    local changed = false
    for _, del in ipairs(tombstones) do
        local key = del.dedup_key
        if key and key ~= "" and known[key] ~= nil then
            known[key] = nil
            changed = true
        end
    end
    if changed then
        self.ui.doc_settings:saveSetting("bookhoard_known_keys", known)
    end
    return changed
end

-- Diff the remembered keys against the local annotation store. Returns
-- arrays of { dedup_key = key } entries for highlights (incl. notes) and
-- bookmarks whose annotations vanished locally, or nil when there is nothing
-- to report. Requires a readable local store: with none, local absence is
-- meaningless and nothing may be flagged.
function Bookhoard:collectDeletions()
    local model, entries = self:getAnnotationStore()
    if not model then return nil, nil end
    local known = self:getKnownAnnotationKeys()
    if not known or next(known) == nil then return nil, nil end

    local present = {}
    for _, bm in ipairs(entries) do
        if bm.bookhoard_dedup_key then
            present[bm.bookhoard_dedup_key] = true
        end
    end

    local deleted_highlights, deleted_bookmarks
    for key, kind in pairs(known) do
        if not present[key] then
            if kind == "bookmark" then
                deleted_bookmarks = deleted_bookmarks or {}
                table.insert(deleted_bookmarks, { dedup_key = key })
            else
                deleted_highlights = deleted_highlights or {}
                table.insert(deleted_highlights, { dedup_key = key })
            end
        end
    end
    return deleted_highlights, deleted_bookmarks
end

-- After a successful push the flagged deletions are now tombstones on the
-- server; drop their keys so they are not re-reported forever (the tombstone
-- echo on the next pull would clear them too, this just avoids the wait).
function Bookhoard:pruneKnownAnnotationKeys(deleted_highlights, deleted_bookmarks)
    if not self.ui.doc_settings then return end
    if not deleted_highlights and not deleted_bookmarks then return end
    local known = self:getKnownAnnotationKeys()
    if not known then return end
    local changed = false
    for _, list in ipairs({ deleted_highlights, deleted_bookmarks }) do
        if list then
            for _, entry in ipairs(list) do
                if known[entry.dedup_key] ~= nil then
                    known[entry.dedup_key] = nil
                    changed = true
                end
            end
        end
    end
    if changed then
        self.ui.doc_settings:saveSetting("bookhoard_known_keys", known)
        self.ui.doc_settings:flush()
    end
end

function Bookhoard:collectAnnotations()
    local bookmarks = json_util.InitArray({})
    local highlights = json_util.InitArray({})
    local notes = json_util.InitArray({})

    local model, entries = self:getAnnotationStore()
    if not model then
        return bookmarks, highlights, notes
    end

    local file_sha256 = self:getFileSHA256()
    local total_pages = self.ui.document:getPageCount()
    local has_pages = self.ui.document.info.has_pages

    local function wirePos(p, fallback_page)
        -- v2 pdf positions are tables {page=N,x,y}; the wire format is a
        -- string — the page number is the durable locator there.
        if type(p) == "table" then
            return tostring(p.page or fallback_page or "")
        end
        return p or ""
    end

    local function echoColor(bm)
        -- Color the device can vouch for as user intent:
        --  - entries we applied ourselves carry the device DEFAULT color
        --    (set at apply time); pushing it back would clobber the web
        --    color. Suppress it unless the user has edited the entry since
        --    (datetime_updated — set by KOReader on any modification).
        --  - device-native entries carry a user-chosen (or default) color;
        --    those push their color as before.
        if bm.bookhoard_dedup_key and not bm.datetime_updated then
            return nil
        end
        return bm.color
    end

    for _, bm in ipairs(entries) do
        local sel_text, user_note
        if model == "v2" then
            if bm.drawer then
                sel_text = bm.text or ""  -- highlighted text
                user_note = bm.note or "" -- user note
            else
                -- Page bookmark. `drawer` is the only reliable highlight
                -- discriminator: KOReader auto-fills text = "in Chapter X"
                -- on bookmarks (updateItemByXPointer), so text-presence
                -- would misclassify every bookmark as a highlight on echo.
                -- The user label lives in `note`.
                sel_text = ""
                user_note = bm.note or ""
            end
        else
            sel_text = bm.notes or ""   -- v1: notes held the highlighted text
            user_note = bm.text or ""   -- v1: text held the note/label
        end

        -- Thin-client policy: no per-annotation CRE lookups here (a
        -- getPageFromXPointer call each is the most expensive thing this
        -- loop can do on weak hardware). Percentages are arithmetic for
        -- paging documents and simply omitted for CRE documents — the
        -- server derives them from the locator against the actual book.
        local page_num = tonumber(bm.page) or 0
        local percentage = has_pages and total_pages > 0 and (page_num / total_pages) or nil

        local chapter = bm.chapter or ""
        if type(chapter) == "table" then chapter = "" end

        local entry = {
            chapter = chapter,
            datetime = bm.datetime or "",
            notes = user_note,
            pos0 = wirePos(bm.pos0, bm.page),
            pos1 = wirePos(bm.pos1, bm.page),
            page = tostring(bm.page or ""),
            text = sel_text,
            book_sha256 = file_sha256,
        }
        if percentage then
            entry.percentage = Math.roundPercent(percentage)
        end
        if bm.bookhoard_dedup_key then
            -- Echo identity for entries received from the server: lets the
            -- server match this push to the original row instead of
            -- minting a duplicate (device locators ≠ web locators, so the
            -- computed key would never match).
            entry.dedup_key = bm.bookhoard_dedup_key
        end

        local has_text = sel_text ~= ""
        local has_notes = user_note ~= ""

        if has_text and has_notes then
            entry.type = "note"
            table.insert(notes, entry)
        elseif has_text then
            entry.type = "highlight"
            local ec = echoColor(bm)
            if ec then
                entry.color = ec
            end
            table.insert(highlights, entry)
        else
            entry.type = "bookmark"
            entry.text = user_note -- bookmark label travels in `text`
            table.insert(bookmarks, entry)
        end
    end

    return bookmarks, highlights, notes
end

function Bookhoard:syncToProgress(progress, percentage)
    logger.dbg("Bookhoard: sync to progress", progress, percentage)
    if self.ui.document.info.has_pages then
        local page = tonumber(progress)
        if page then
            self.ui:handleEvent(Event:new("GotoPage", page))
        end
    elseif progress and progress:match("^/body/") then
        self.ui:handleEvent(Event:new("GotoXPointer", progress))
    elseif percentage then
        -- Navigation goes through events (ReaderRolling/ReaderPaging both
        -- implement onGotoPercent and refresh the view themselves). Calling
        -- document:gotoPercent directly crashed KOReader — no such method
        -- exists on documents.
        self.ui:handleEvent(Event:new("GotoPercent", percentage * 100))
    end
end

function Bookhoard:updateProgress(ensure_networking, interactive)
    if not self:isConfigured() then
        if interactive then
            UIManager:show(InfoMessage:new{
                text = _("Please configure and register your device first."),
                timeout = 3,
            })
        end
        return
    end

    if not self.settings.sync_progress then
        if interactive then
            UIManager:show(InfoMessage:new{
                text = _("Reading progress sync is disabled (see What to sync)."),
                timeout = 3,
            })
        end
        return
    end

    if not self.ui.document then
        if interactive then
            UIManager:show(InfoMessage:new{
                text = _("No document open."),
                timeout = 3,
            })
        end
        return
    end

    local now = UIManager:getElapsedTimeSinceBoot()
    if not interactive and now - self.push_timestamp <= API_CALL_DEBOUNCE_DELAY then
        return
    end

    if ensure_networking then
        -- Same silent-drop case as getProgress: connected but no internet
        -- means willRerunWhenOnline neither reruns nor shows any UI.
        if interactive and NetworkMgr:isConnected() and not NetworkMgr:isOnline() then
            UIManager:show(InfoMessage:new{
                text = _("No internet connection. Progress was not pushed."),
                timeout = 3,
            })
            return
        end
        if NetworkMgr:willRerunWhenOnline(function() self:updateProgress(ensure_networking, interactive) end) then
            return
        end
    end

    UIManager:scheduleIn(0.5, function()
        self:_doUpdateProgress(interactive)
    end)

    self.push_timestamp = now
end

function Bookhoard:_doUpdateProgress(interactive)
    local book_data = self:collectBookData()
    if not book_data then return end

    if self.settings.sync_bookmarks or self.settings.sync_highlights or self.settings.sync_notes then
        local bm, hl, nt = self:collectAnnotations()
        book_data.bookmarks = self.settings.sync_bookmarks and bm or {}
        book_data.highlights = self.settings.sync_highlights and hl or {}
        book_data.notes = self.settings.sync_notes and nt or {}
    else
        book_data.bookmarks = {}
        book_data.highlights = {}
        book_data.notes = {}
    end

    -- Report annotations deleted locally since the last pull. Gated per
    -- category exactly like the arrays above: a disabled category sends
    -- neither its annotations nor its deletions.
    local deleted_hl, deleted_bm = self:collectDeletions()
    if deleted_hl then
        if self.settings.sync_highlights then
            book_data.deleted_highlights = deleted_hl
        else
            deleted_hl = nil -- not sent: must not be pruned below
        end
    end
    if deleted_bm then
        if self.settings.sync_bookmarks then
            book_data.deleted_bookmarks = deleted_bm
        else
            deleted_bm = nil
        end
    end

    local api = self:getAPI()
    local ok, result = api:syncProgress(book_data, self.settings.sync_mode)

    UIManager:nextTick(function()
        if ok then
            logger.dbg("Bookhoard: progress pushed successfully")
            if result and result.book_results then
                for _, br in ipairs(result.book_results) do
                    if br.synced and br.book_uuid and br.sha256 == book_data.sha256 then
                        self.ui.doc_settings:saveSetting("bookhoard_uuid", br.book_uuid)
                        self.ui.doc_settings:flush()
                        break
                    end
                end
            end
            self:pruneKnownAnnotationKeys(deleted_hl, deleted_bm)
            if interactive then
                UIManager:show(InfoMessage:new{
                    text = _("Progress has been pushed."),
                    timeout = 3,
                })
            end
        else
            logger.warn("Bookhoard: failed to push progress")
            if interactive then
                UIManager:show(InfoMessage:new{
                    text = _("Failed to push progress. Check your network connection."),
                    timeout = 3,
                })
            end
        end
    end)
end

function Bookhoard:getProgress(ensure_networking, interactive)
    if not self:isConfigured() then
        if interactive then
            UIManager:show(InfoMessage:new{
                text = _("Please configure and register your device first."),
                timeout = 3,
            })
        end
        return
    end

    if not self.settings.sync_progress then
        if interactive then
            UIManager:show(InfoMessage:new{
                text = _("Reading progress sync is disabled (see What to sync)."),
                timeout = 3,
            })
        end
        return
    end

    local now = UIManager:getElapsedTimeSinceBoot()
    if not interactive and now - self.pull_timestamp <= API_CALL_DEBOUNCE_DELAY then
        return
    end

    if ensure_networking then
        -- willRerunWhenOnline silently drops the action (no rerun, no UI)
        -- when the device is connected at the IP level but has no internet
        -- access: it only guarantees isConnected, and in that state the
        -- framework re-connects without ever invoking the callback. Catch
        -- that state up front so an interactive pull never no-ops silently.
        if interactive and NetworkMgr:isConnected() and not NetworkMgr:isOnline() then
            UIManager:show(InfoMessage:new{
                text = _("No internet connection. Progress was not pulled."),
                timeout = 3,
            })
            return
        end
        if NetworkMgr:willRerunWhenOnline(function() self:getProgress(ensure_networking, interactive) end) then
            return
        end
    end

    local book_uuid = self:getBookhoardUUID()
    if not book_uuid then
        -- No cached UUID yet (first open of a freshly downloaded book). Link
        -- the book with a read-only SHA-256 lookup, then pull. Progress is
        -- never pushed here: the device sits on the first page of a book the
        -- server may hold mid-read, and pushing that would overwrite/server-
        -- conflict the real progress before the pull could deliver it.
        if interactive then
            UIManager:show(InfoMessage:new{
                text = _("Linking this book to Bookhoard first…"),
                timeout = 3,
            })
        end
        self:_linkBookThenPull(interactive)
        return
    end

    UIManager:scheduleIn(0.5, function()
        self:_doGetProgress(interactive)
    end)

    self.pull_timestamp = now
end

-- Link the current document to Bookhoard WITHOUT pushing progress, then run
-- the pull. The UUID is learned via a read-only SHA-256 resolve; only when
-- the server does not know the hash at all (sideloaded book) do we fall back
-- to the legacy push-bootstrap, and only for background (non-interactive)
-- flows — a user-initiated pull must never push this device's position.
function Bookhoard:_linkBookThenPull(interactive)
    local file_sha256 = self:getFileSHA256()
    if not file_sha256 then
        if interactive then
            UIManager:show(InfoMessage:new{
                text = _("Could not read this file to link it with Bookhoard."),
                timeout = 3,
            })
        end
        return
    end

    UIManager:scheduleIn(0.5, function()
        local api = self:getAPI()
        local ok, result = api:resolveBook(file_sha256)

        UIManager:nextTick(function()
            if not self.ui.doc_settings then return end

            if ok and type(result) == "table" and result.book_uuid then
                self.ui.doc_settings:saveSetting("bookhoard_uuid", result.book_uuid)
                self.ui.doc_settings:flush()
                logger.dbg("Bookhoard: linked book by SHA-256 to", result.book_uuid)
                self:_doGetProgress(interactive)
                return
            end

            if result and result.status == 404 then
                -- Two very different failures share this status. The server's
                -- resolve handler answers a book miss with JSON
                -- {"error": "book not found"}; anything else behind a 404
                -- (an HTML error page) means the route itself is missing —
                -- a server image predating the resolve endpoint, a reverse
                -- proxy, or a wrong server URL. Only the former may advise
                -- pushing to link: acting on a stale-server 404 would push
                -- this device's first-page position over the server's real
                -- progress — the very conflict this flow exists to prevent.
                if result.error == "book not found" then
                    -- Genuinely unknown to the server (sideloaded book). The
                    -- legacy push-bootstrap only ever linked such books via
                    -- fuzzy title/author matching; keep that as a background
                    -- attempt, but tell interactive users to push explicitly.
                    if interactive then
                        UIManager:show(InfoMessage:new{
                            text = _("This book is not in your Bookhoard library yet. Push progress to link it."),
                            timeout = 4,
                        })
                    else
                        self:_bootstrapUUIDThenPull(interactive)
                    end
                else
                    logger.warn("Bookhoard: resolve endpoint returned 404 without a book-not-found body (stale server?)")
                    if interactive then
                        UIManager:show(InfoMessage:new{
                            text = _("Your Bookhoard server does not support this request and may be outdated. Update the server and try again."),
                            timeout = 5,
                        })
                    end
                end
                return
            end

            logger.warn("Bookhoard: failed to resolve book by SHA-256")
            if interactive then
                UIManager:show(InfoMessage:new{
                    text = _("Could not reach Bookhoard. Check your network connection and try again."),
                    timeout = 4,
                })
            end
        end)
    end)
end

-- Legacy fallback: establish the bookhoard UUID for the current document by
-- pushing once, then (once cached) perform the originally-requested pull.
-- Only reached from background flows when the SHA-256 resolve reported the
-- book as unknown to the server; the push mainly serves fuzzy title/author
-- linking and device alias registration for sideloaded books.
function Bookhoard:_bootstrapUUIDThenPull(interactive)
    self:updateProgress(false, false)
    UIManager:scheduleIn(2, function()
        if self:getBookhoardUUID() then
            self:_doGetProgress(interactive)
        elseif interactive then
            UIManager:show(InfoMessage:new{
                text = _("Could not link this book. Check your network and try again."),
                timeout = 3,
            })
        end
    end)
end

function Bookhoard:_doGetProgress(interactive)
    local book_uuid = self:getBookhoardUUID()
    if not book_uuid then return end

    local api = self:getAPI()
    local ok, result = api:getMetadata(book_uuid)

    UIManager:nextTick(function()
        if not ok or not result then
            if interactive then
                UIManager:show(InfoMessage:new{
                    text = _("Failed to pull progress. Check your network connection."),
                    timeout = 3,
                })
            end
            return
        end

        if not result.progress then
            if interactive then
                UIManager:show(InfoMessage:new{
                    text = _("No progress found for this document."),
                    timeout = 3,
                })
            end
            return
        end

        if result.annotations then
            self:applyServerAnnotations(result.annotations)
        end

        local progress = result.progress
        local percentage = self:getLastPercent()
        local server_percentage = progress.percentage or 0

        if percentage == server_percentage then
            if interactive then
                UIManager:show(InfoMessage:new{
                    text = _("Progress is already synchronized."),
                    timeout = 3,
                })
            end
            return
        end

        local self_older = server_percentage > percentage

        local nav_target
        local sync_text
        if self.ui.document.info.has_pages then
            -- Fixed-layout: the page index is the canonical locator. CFI/xpointer
            -- are meaningless for image-based content, so use progress.page directly.
            local total = progress.total_pages or self.ui.document:getPageCount()
            local target_page = progress.page
                or math.min(Math.round(server_percentage * total), total)
            nav_target = progress.page
            sync_text = T(_("Sync to page %1 of %2 from server?"), target_page, total)
        else
            nav_target = progress.koreader_xpointer or progress.epubcfi or progress.page
            local total = self.ui.document:getPageCount()
            local target_page = math.min(Math.round(server_percentage * total), total)
            sync_text = T(_("Sync to page %1 of %2 from server?"), target_page, total)
        end

        if self_older then
            if self.settings.sync_forward == SYNC_STRATEGY.SILENT then
                self:syncToProgress(nav_target, server_percentage)
                self:_showSyncedMessage()
            elseif self.settings.sync_forward == SYNC_STRATEGY.PROMPT then
                UIManager:show(ConfirmBox:new{
                    text = sync_text,
                    ok_callback = function()
                        self:syncToProgress(nav_target, server_percentage)
                    end,
                })
            end
        else
            if self.settings.sync_backward == SYNC_STRATEGY.SILENT then
                self:syncToProgress(nav_target, server_percentage)
                self:_showSyncedMessage()
            elseif self.settings.sync_backward == SYNC_STRATEGY.PROMPT then
                UIManager:show(ConfirmBox:new{
                    text = sync_text,
                    ok_callback = function()
                        self:syncToProgress(nav_target, server_percentage)
                    end,
                })
            end
        end
    end)
end

function Bookhoard:applyServerAnnotations(annotations)
    if not annotations then return end
    local model, entries = self:getAnnotationStore()
    if not model then return end

    local changed = false
    local notify_events = {}
    local function notify(ev)
        notify_events[#notify_events + 1] = ev
    end
    local has_pages = self.ui.document.info.has_pages

    local function findLocal(server_entry, has_text)
        -- Identity match by dedup key: survives pos0 drift (improved
        -- server conversion) and, crucially, never cross-matches a
        -- DIFFERENT annotation that merely shares the position.
        local key = server_entry.dedup_key
        if key and key ~= "" then
            for i, bm in ipairs(entries) do
                if bm.bookhoard_dedup_key == key then
                    return i
                end
            end
            -- No key holder, but a keyed serve may still be THIS device's
            -- own creation echoed back: the server stamped a computed key
            -- on our push before we ever pulled one. Adopt a keyless local
            -- annotation of the same kind at the identical pos0
            -- (addOrUpdate stamps + updates it) instead of applying a
            -- duplicate beside it. Keyed local entries are never
            -- pos0-matched, so distinct annotations sharing a position
            -- cannot cross-match.
            local pos0 = server_entry.pos0 or ""
            if pos0 ~= "" then
                for i, bm in ipairs(entries) do
                    if not bm.bookhoard_dedup_key then
                        local is_highlight
                        if model == "v2" then
                            is_highlight = bm.drawer ~= nil
                        else
                            is_highlight = bm.notes ~= nil and bm.notes ~= ""
                        end
                        if is_highlight ~= has_text then
                            -- Wrong kind: a page bookmark at the same spot
                            -- is a different annotation, never an echo.
                        else
                            local p = bm.pos0
                            if type(p) == "table" then p = tostring(p.page or "") end
                            if p == pos0 then
                                return i
                            end
                        end
                    end
                end
            end
            return nil -- genuinely absent: a real new entry
        end
        -- Legacy serve (no key): fall back to pos0 matching.
        local pos0 = server_entry.pos0 or ""
        if pos0 == "" then return nil end
        for i, bm in ipairs(entries) do
            local p = bm.pos0
            if type(p) == "table" then p = tostring(p.page or "") end
            if p == pos0 then
                return i
            end
        end
        return nil
    end

    -- Get/set the highlighted text and the user note with model-appropriate
    -- field names (v2: text/note; v1: notes/text).
    local function getFields(bm)
        if model == "v2" then
            return bm.text or "", bm.note or ""
        end
        return bm.notes or "", bm.text or ""
    end
    local function setFields(bm, sel_text, user_note)
        if model == "v2" then
            bm.text = sel_text ~= "" and sel_text or nil
            bm.note = user_note ~= "" and user_note or nil
        else
            bm.notes = sel_text
            bm.text = user_note
        end
    end

    local function isValidPos0(pos0)
        -- Rolling (CRE) documents address bookmarks/highlights by xpointer;
        -- paging documents (PDF/comics/DjVu) by page number. Anything else
        -- (a raw "cfi:" locator, an epubcfi(...) string, a JSON anchor, …)
        -- cannot be placed in this document and must not become a local
        -- bookmark: it could never be matched again and would be re-pushed
        -- to the server as a junk duplicate on the next sync.
        if pos0 == "" then return false end
        if has_pages then
            return tonumber(pos0) ~= nil
        end
        return pos0:sub(1, 6) == "/body/"
    end

    local function makePageFromPos(pos0)
        if has_pages then
            local page = tonumber(pos0:match("(%d+)$"))
            return page or pos0
        end
        return pos0
    end

    -- v2 pdf positions are {page=N} tables; CRE keeps xpointer strings.
    local function makeLocalPos(pos)
        if has_pages then
            local page = tonumber(pos)
            return page and { page = page } or nil
        end
        return pos
    end

    local function addOrUpdate(server_entry, has_text)
        local pos0 = server_entry.pos0 or ""
        if not isValidPos0(pos0) then return end
        local srv_text = server_entry.text or ""
        local srv_notes = server_entry.notes or ""
        local srv_key = server_entry.dedup_key
        -- Annotations that exist locally (after this call) get their key
        -- remembered so a later deletion can be reported. Unplaceable
        -- locators return above and are never tracked — absence of an
        -- annotation this device never held is not a deletion.
        local kind = has_text and "highlight" or "bookmark"
        local idx = findLocal(server_entry, has_text)
        if idx then
            local bm = entries[idx]
            -- Legacy pos0 match on a device-native entry: stamp the key so
            -- identity matching and deletion tracking work from now on.
            if srv_key and srv_key ~= "" and not bm.bookhoard_dedup_key then
                bm.bookhoard_dedup_key = srv_key
                changed = true
            end
            changed = self:rememberAnnotationKey(kind, srv_key) or changed
            local cur_text, cur_note = getFields(bm)
            if cur_text ~= srv_text or cur_note ~= srv_notes then
                setFields(bm, srv_text, srv_notes)
                if server_entry.color and server_entry.color ~= "" then bm.color = server_entry.color end
                notify({ bm })
                changed = true
            end
        elseif model == "v2" then
            -- `drawer` marks a renderable highlight in v2; its absence
            -- makes the entry a page bookmark (label goes in `note`).
            local entry = {
                datetime = server_entry.datetime ~= "" and server_entry.datetime or nil,
                chapter = server_entry.chapter ~= "" and server_entry.chapter or nil,
                pos0 = makeLocalPos(pos0),
                pos1 = makeLocalPos(server_entry.pos1 ~= "" and server_entry.pos1 or pos0),
                page = makePageFromPos(pos0),
            }
            if server_entry.dedup_key and server_entry.dedup_key ~= "" then
                entry.bookhoard_dedup_key = server_entry.dedup_key
            end
            if has_text then
                entry.drawer = "lighten"
                entry.text = srv_text
                if srv_notes ~= "" then entry.note = srv_notes end
                -- Color: the server maps the web color into KOReader's
                -- named palette — use it when it arrives as a plain name so
                -- synced highlights keep their web color. Anything else
                -- (absent, or a raw hex that slipped through) falls back to
                -- the device default (view.highlight.saved_color, "yellow"
                -- on color screens — nil would render grey via darkenRect).
                -- Echoes of applied entries never push this color back
                -- (collectAnnotations) unless the user edited them, so the
                -- stored web color never drifts.
                local srv_color = server_entry.color or ""
                if srv_color ~= "" and not srv_color:find("^#") then
                    entry.color = srv_color
                else
                    entry.color = (self.ui.view and self.ui.view.highlight
                        and self.ui.view.highlight.saved_color) or "yellow"
                end
            else
                if srv_text ~= "" then entry.note = srv_text end -- bookmark label
            end
            local index = self.ui.annotation:addItem(entry)
            notify({ entry, index_modified = index })
            changed = self:rememberAnnotationKey(kind, srv_key) or changed
            changed = true
        else
            local entry = {
                page = makePageFromPos(pos0),
                pos0 = pos0,
                pos1 = server_entry.pos1 or pos0,
                datetime = server_entry.datetime or "",
                notes = srv_text,
                text = srv_notes,
            }
            if has_text then
                local srv_color = server_entry.color or ""
                if srv_color ~= "" and not srv_color:find("^#") then
                    entry.color = srv_color
                end
            end
            if server_entry.dedup_key and server_entry.dedup_key ~= "" then
                entry.bookhoard_dedup_key = server_entry.dedup_key
            end
            if server_entry.chapter and server_entry.chapter ~= "" then entry.chapter = server_entry.chapter end
            table.insert(entries, entry)
            changed = self:rememberAnnotationKey(kind, srv_key) or changed
            changed = true
        end
    end

    if self.settings.sync_highlights and annotations.highlights then
        for _, hl in ipairs(annotations.highlights) do
            addOrUpdate(hl, true)
        end
    end

    if self.settings.sync_notes and annotations.notes then
        for _, note in ipairs(annotations.notes) do
            addOrUpdate(note, true)
        end
    end

    if self.settings.sync_bookmarks and annotations.bookmarks then
        for _, bm in ipairs(annotations.bookmarks) do
            addOrUpdate(bm, false)
        end
    end

    local function findLocalByTombstone(del, want_highlight)
        local key = del.dedup_key
        if key and key ~= "" then
            for i, bm in ipairs(entries) do
                if bm.bookhoard_dedup_key == key then
                    return i
                end
            end
            return nil -- keyed tombstone: only its own entry may be removed
        end
        -- Legacy tombstone (no key): pos0 match, restricted to the same
        -- annotation kind so a highlight tombstone never eats a bookmark
        -- that shares the position.
        local pos0 = del.pos0 or ""
        if pos0 == "" then return nil end
        for i, bm in ipairs(entries) do
            local p = bm.pos0
            if type(p) == "table" then p = tostring(p.page or "") end
            if p == pos0 and (not not bm.drawer) == want_highlight then
                return i
            end
        end
        return nil
    end

    if annotations.deleted_highlights then
        changed = self:forgetAnnotationKeys(annotations.deleted_highlights) or changed
        for _, del in ipairs(annotations.deleted_highlights) do
            local idx = findLocalByTombstone(del, true)
            local bm = idx and entries[idx]
            local cur_text = bm and (getFields(bm)) or ""
            if idx and cur_text ~= "" then
                table.remove(entries, idx)
                notify({ bm, index_modified = -idx })
                changed = true
            end
        end
    end

    if annotations.deleted_bookmarks then
        changed = self:forgetAnnotationKeys(annotations.deleted_bookmarks) or changed
        for _, del in ipairs(annotations.deleted_bookmarks) do
            local idx = findLocalByTombstone(del, false)
            if idx then
                local removed = entries[idx]
                table.remove(entries, idx)
                notify({ removed, index_modified = -idx })
                changed = true
            end
        end
    end

    if changed then
        -- addItem() inserts v2 entries at their sorted position already;
        -- explicit re-sorting is a v1-model need only.
        if model == "v1" and self.ui.bookmark and self.ui.bookmark.onSortBookmarks then
            self.ui.bookmark:onSortBookmarks()
        end
        if self.ui.saveSettings then
            self.ui:saveSettings()
        end
        -- ReaderView caches each rendered page's highlight boxes and only
        -- invalidates them on AnnotationsModified — without this, applied
        -- annotations don't paint until restart despite the repaint.
        -- Payloads MUST be tables: ReaderThumbnail iterates them, and a
        -- nil payload crashed KOReader (#nil) on pull. Per-item payloads
        -- with index_modified mirror the native add/update/remove
        -- dispatches exactly (page-level cache invalidation + cached box
        -- index shifting).
        for _, ev in ipairs(notify_events) do
            self.ui:handleEvent(Event:new("AnnotationsModified", ev))
        end
        if self.ui.dialog then
            UIManager:setDirty(self.ui.dialog, "ui")
        end
        logger.dbg("Bookhoard: annotations updated from server")
    end
end

function Bookhoard:_showSyncedMessage()
    UIManager:show(InfoMessage:new{
        text = _("Progress has been synchronized."),
        timeout = 3,
    })
end

function Bookhoard:_onCloseDocument()
    self.onResume = nil
    self.onSuspend = nil
    NetworkMgr:goOnlineToRun(function()
        self:updateProgress(false, false)
    end)
end

function Bookhoard:schedulePeriodicPush()
    UIManager:unschedule(self.periodic_push_task)
    UIManager:scheduleIn(PERIODIC_PUSH_DELAY, self.periodic_push_task)
    self.periodic_push_scheduled = true
end

function Bookhoard:_onPageUpdate(page)
    if page == nil then return end

    if self.last_page ~= page then
        self.last_page = page
        self.page_update_counter = self.page_update_counter + 1
        if self.periodic_push_scheduled
            or (self.settings.pages_before_update
                and self.page_update_counter >= self.settings.pages_before_update) then
            self:schedulePeriodicPush()
        end
    end
end

function Bookhoard:_onResume()
    if Device:hasWifiRestore() and NetworkMgr.wifi_was_on
        and G_reader_settings:isTrue("auto_restore_wifi") then
        return
    end
    UIManager:scheduleIn(1, function()
        self:getProgress(true, false)
    end)
end

function Bookhoard:_onSuspend()
    self:updateProgress(true, false)
end

function Bookhoard:_onNetworkConnected()
    UIManager:scheduleIn(0.5, function()
        self:getProgress(false, false)
    end)
end

function Bookhoard:_onNetworkDisconnecting()
    self:updateProgress(false, false)
end

function Bookhoard:onCloseWidget()
    UIManager:unschedule(self.periodic_push_task)
    self.periodic_push_task = nil
end

return Bookhoard