bookhoard.koplugin/main.lua

local ConfirmBox = require("ui/widget/confirmbox")
local DataStorage = require("datastorage")
local Device = require("device")
local Event = require("ui/event")
local InfoMessage = require("ui/widget/infomessage")
local InputDialog = require("ui/widget/inputdialog")
local LuaSettings = require("luasettings")
local Math = require("optmath")
local NetworkMgr = require("ui/network/manager")
local SpinWidget = require("ui/widget/spinwidget")
local UIManager = require("ui/uimanager")
local WidgetContainer = require("ui/widget/container/widgetcontainer")
local json = require("json")
local json_util = require("json.util")
local logger = require("logger")
local sha2 = require("ffi/sha2")
local time = require("ui/time")
local util = require("util")
local lfs = require("libs/libkoreader-lfs")
local T = require("ffi/util").template
local _ = require("gettext")

local BookhoardAPI = require("BookhoardAPI")

local SYNC_STRATEGY = {
    PROMPT = 1,
    SILENT = 2,
    DISABLE = 3,
}

local SYNC_MODE = {
    IMMEDIATE = "immediate",
    CHECKPOINT = "checkpoint",
}

local API_CALL_DEBOUNCE_DELAY = time.s(25)
local PERIODIC_PUSH_DELAY = 10

local sha256hex = sha2.sha256hex or sha2.sha256

local Bookhoard = WidgetContainer:extend({
    name = "bookhoard",
    is_doc_only = false,
    title = _("Bookhoard Server"),
    settings_key = "bookhoard",

    push_timestamp = nil,
    pull_timestamp = nil,
    page_update_counter = nil,
    last_page = nil,
    periodic_push_task = nil,
    periodic_push_scheduled = nil,
    registration_poll_scheduled = nil,

    settings = nil,
})

Bookhoard.default_settings = {
    server_url = nil,
    auth_token = nil,
    device_id = nil,
    auto_sync = false,
    pages_before_update = 50,
    sync_forward = SYNC_STRATEGY.PROMPT,
    sync_backward = SYNC_STRATEGY.DISABLE,
    sync_progress = true,
    sync_bookmarks = true,
    sync_highlights = true,
    sync_notes = true,
    sync_mode = SYNC_MODE.IMMEDIATE,
}

function Bookhoard:init()
    self.push_timestamp = 0
    self.pull_timestamp = 0
    self.page_update_counter = 0
    self.last_page = -1
    self.periodic_push_scheduled = false
    self.registration_poll_scheduled = false

    self.periodic_push_task = function()
        self.periodic_push_scheduled = false
        self.page_update_counter = 0
        self:updateProgress(false, false)
    end

    -- readSetting returns the STORED table verbatim when one exists — it
    -- does not merge defaults. A settings file saved by an older plugin
    -- version lacks the annotation toggles entirely, so sync_bookmarks/
    -- highlights/notes read as nil, the "or" in _doUpdateProgress fell
    -- through to the else branch, and every push carried EMPTY annotation
    -- arrays while progress synced fine. Merge missing defaults (and
    -- persist once so the file and the menu checkboxes self-heal).
    self.settings = G_reader_settings:readSetting(self.settings_key, {})
    if type(self.settings) ~= "table" then
        self.settings = {}
    end
    local settings_merged = false
    for key, value in pairs(self.default_settings) do
        if self.settings[key] == nil then
            self.settings[key] = value
            settings_merged = true
        end
    end
    if settings_merged then
        G_reader_settings:saveSetting(self.settings_key, self.settings)
        logger.info("Bookhoard: merged new default settings (older settings file)")
    end

    if self.settings.auto_sync
        and Device:hasSeamlessWifiToggle()
        and G_reader_settings:readSetting("wifi_enable_action") ~= "turn_on" then
        self.settings.auto_sync = false
        logger.warn("Bookhoard: auto-sync disabled because wifi_enable_action is not turn_on")
    end

    self.ui.menu:registerToMainMenu(self)
    self:setupMenuOrder()
end

function Bookhoard:onReaderReady()
    if self.settings.auto_sync then
        UIManager:nextTick(function()
            -- Pull-first on every open. A freshly downloaded book has no cached
            -- bookhoard UUID yet; getProgress resolves it by SHA-256 with a
            -- read-only lookup (no progress push). The previous push-to-
            -- bootstrap strategy transmitted the device's first-page position
            -- to the server on books already mid-read elsewhere, creating a
            -- sync conflict before the first pull could ever happen.
            self:getProgress(true, false)
        end)
    end
    self:registerEvents()
    self.last_page = self.ui:getCurrentPage()
end

function Bookhoard:registerEvents()
    if self.settings.auto_sync then
        self.onCloseDocument = self._onCloseDocument
        self.onPageUpdate = self._onPageUpdate
        self.onResume = self._onResume
        self.onSuspend = self._onSuspend
        self.onNetworkConnected = self._onNetworkConnected
        self.onNetworkDisconnecting = self._onNetworkDisconnecting
    else
        self.onCloseDocument = nil
        self.onPageUpdate = nil
        self.onResume = nil
        self.onSuspend = nil
        self.onNetworkConnected = nil
        self.onNetworkDisconnecting = nil
    end
end

function Bookhoard:getAPI()
    return BookhoardAPI:new({
        server_url = self.settings.server_url,
        auth_token = self.settings.auth_token,
    })
end

function Bookhoard:isConfigured()
    return self.settings.server_url and self.settings.auth_token
end

function Bookhoard:getSyncPeriod()
    if not self.settings.auto_sync then
        return _("Not available")
    end
    local period = self.settings.pages_before_update
    if period and period > 0 then
        return period
    end
    return _("Never")
end

function Bookhoard:addToMainMenu(menu_items)
    menu_items.bookhoard_sync = {
        text = _("Bookhoard sync"),
        sorting_hint = "tools",
        sub_item_table = self:buildMainMenu(),
    }
end

function Bookhoard:buildMainMenu()
    local items = {}

    if self:isConfigured() then
        table.insert(items, {
            text = _("Push progress from this device"),
            callback = function()
                self:updateProgress(true, true)
            end,
        })
        table.insert(items, {
            text = _("Pull progress from server"),
            callback = function()
                self:getProgress(true, true)
            end,
            separator = true,
        })
    end

    table.insert(items, {
        text = _("Server URL"),
        keep_menu_open = true,
        tap_input_func = function()
            return {
                title = _("Bookhoard server URL"),
                input = self.settings.server_url or "http://",
                callback = function(input)
                    self.settings.server_url = input ~= "" and input or nil
                    G_reader_settings:saveSetting(self.settings_key, self.settings)
                end,
            }
        end,
    })

    if self:isConfigured() then
        table.insert(items, {
            text = _("Device info"),
            keep_menu_open = true,
            callback = function()
                UIManager:show(InfoMessage:new{
                    text = T(_("Device ID: %1\nServer: %2"),
                        self.settings.device_id or _("unknown"),
                        self.settings.server_url),
                })
            end,
        })
        table.insert(items, {
            text = _("Disconnect"),
            keep_menu_open = true,
            callback = function()
                UIManager:show(ConfirmBox:new{
                    text = _("Disconnect from Bookhoard server?"),
                    ok_text = _("Disconnect"),
                    ok_callback = function()
                        self.settings.auth_token = nil
                        self.settings.device_id = nil
                        self.settings.auto_sync = false
                        G_reader_settings:saveSetting(self.settings_key, self.settings)
                        self:registerEvents()
                        UIManager:askForRestart()
                    end,
                })
            end,
            separator = true,
        })
    else
        table.insert(items, {
            text = _("Register device"),
            keep_menu_open = true,
            callback = function()
                self:startRegistration()
            end,
            separator = true,
        })
    end

    table.insert(items, {
        text = _("Automatically push progress"),
        checked_func = function() return self.settings.auto_sync end,
        help_text = _([[This may lead to prompts about toggling WiFi on document close and suspend/resume, depending on your device's connectivity.]]),
        callback = function()
            self:toggleAutoSync()
        end,
    })

    table.insert(items, {
        text_func = function()
            return T(_("Periodically sync every # pages (%1)"), self:getSyncPeriod())
        end,
        enabled_func = function() return self.settings.auto_sync end,
        keep_menu_open = true,
        callback = function(touchmenu_instance)
            local spin = SpinWidget:new{
                text = _([[Number of page turns between progress updates. Set to 0 to disable.]]),
                value = self.settings.pages_before_update or 0,
                value_min = 0,
                value_max = 999,
                value_step = 1,
                value_hold_step = 10,
                ok_text = _("Set"),
                title_text = _("Pages before update"),
                default_value = 50,
                callback = function(spin)
                    self.settings.pages_before_update = spin.value > 0 and spin.value or nil
                    G_reader_settings:saveSetting(self.settings_key, self.settings)
                    if touchmenu_instance then touchmenu_instance:updateItems() end
                end,
            }
            UIManager:show(spin)
        end,
    })

    table.insert(items, {
        text_func = function()
            return T(_("Sync mode (%1)"),
                self.settings.sync_mode == SYNC_MODE.IMMEDIATE and _("immediate") or _("checkpoint"))
        end,
        sub_item_table = {
            {
                text = _("Immediate"),
                checked_func = function()
                    return self.settings.sync_mode == SYNC_MODE.IMMEDIATE
                end,
                callback = function()
                    self.settings.sync_mode = SYNC_MODE.IMMEDIATE
                    G_reader_settings:saveSetting(self.settings_key, self.settings)
                end,
            },
            {
                text = _("Checkpoint"),
                checked_func = function()
                    return self.settings.sync_mode == SYNC_MODE.CHECKPOINT
                end,
                callback = function()
                    self.settings.sync_mode = SYNC_MODE.CHECKPOINT
                    G_reader_settings:saveSetting(self.settings_key, self.settings)
                end,
            },
        },
        separator = true,
    })

    table.insert(items, {
        text = _("Sync behavior"),
        sub_item_table = {
            {
                text_func = function()
                    return T(_("Sync to a newer state (%1)"),
                        self:getStrategyName(self.settings.sync_forward))
                end,
                sub_item_table = {
                    {
                        text = _("Silently"),
                        checked_func = function()
                            return self.settings.sync_forward == SYNC_STRATEGY.SILENT
                        end,
                        callback = function()
                            self.settings.sync_forward = SYNC_STRATEGY.SILENT
                            G_reader_settings:saveSetting(self.settings_key, self.settings)
                        end,
                    },
                    {
                        text = _("Prompt"),
                        checked_func = function()
                            return self.settings.sync_forward == SYNC_STRATEGY.PROMPT
                        end,
                        callback = function()
                            self.settings.sync_forward = SYNC_STRATEGY.PROMPT
                            G_reader_settings:saveSetting(self.settings_key, self.settings)
                        end,
                    },
                    {
                        text = _("Never"),
                        checked_func = function()
                            return self.settings.sync_forward == SYNC_STRATEGY.DISABLE
                        end,
                        callback = function()
                            self.settings.sync_forward = SYNC_STRATEGY.DISABLE
                            G_reader_settings:saveSetting(self.settings_key, self.settings)
                        end,
                    },
                },
            },
            {
                text_func = function()
                    return T(_("Sync to an older state (%1)"),
                        self:getStrategyName(self.settings.sync_backward))
                end,
                sub_item_table = {
                    {
                        text = _("Silently"),
                        checked_func = function()
                            return self.settings.sync_backward == SYNC_STRATEGY.SILENT
                        end,
                        callback = function()
                            self.settings.sync_backward = SYNC_STRATEGY.SILENT
                            G_reader_settings:saveSetting(self.settings_key, self.settings)
                        end,
                    },
                    {
                        text = _("Prompt"),
                        checked_func = function()
                            return self.settings.sync_backward == SYNC_STRATEGY.PROMPT
                        end,
                        callback = function()
                            self.settings.sync_backward = SYNC_STRATEGY.PROMPT
                            G_reader_settings:saveSetting(self.settings_key, self.settings)
                        end,
                    },
                    {
                        text = _("Never"),
                        checked_func = function()
                            return self.settings.sync_backward == SYNC_STRATEGY.DISABLE
                        end,
                        callback = function()
                            self.settings.sync_backward = SYNC_STRATEGY.DISABLE
                            G_reader_settings:saveSetting(self.settings_key, self.settings)
                        end,
                    },
                },
            },
        },
        separator = true,
    })

    table.insert(items, {
        text = _("What to sync"),
        sub_item_table = {
            {
                text = _("Reading progress"),
                checked_func = function() return self.settings.sync_progress end,
                callback = function()
                    self.settings.sync_progress = not self.settings.sync_progress
                    G_reader_settings:saveSetting(self.settings_key, self.settings)
                end,
            },
            {
                text = _("Bookmarks"),
                checked_func = function() return self.settings.sync_bookmarks end,
                callback = function()
                    self.settings.sync_bookmarks = not self.settings.sync_bookmarks
                    G_reader_settings:saveSetting(self.settings_key, self.settings)
                end,
            },
            {
                text = _("Highlights"),
                checked_func = function() return self.settings.sync_highlights end,
                callback = function()
                    self.settings.sync_highlights = not self.settings.sync_highlights
                    G_reader_settings:saveSetting(self.settings_key, self.settings)
                end,
            },
            {
                text = _("Notes"),
                checked_func = function() return self.settings.sync_notes end,
                callback = function()
                    self.settings.sync_notes = not self.settings.sync_notes
                    G_reader_settings:saveSetting(self.settings_key, self.settings)
                end,
            },
        },
        separator = true,
    })

    table.insert(items, {
        text = _("Setup OPDS catalog"),
        keep_menu_open = true,
        enabled_func = function() return self:isConfigured() end,
        callback = function()
            if self:setupOPDS() then
                UIManager:show(InfoMessage:new{
                    text = _("Bookhoard OPDS catalog added! Find it in Home → OPDS Catalog."),
                    timeout = 3,
                })
            else
                UIManager:show(InfoMessage:new{
                    text = _("Please configure and register your device first."),
                    timeout = 3,
                })
            end
        end,
    })

    return items
end

function Bookhoard:getStrategyName(strategy)
    if strategy == SYNC_STRATEGY.PROMPT then
        return _("Prompt")
    elseif strategy == SYNC_STRATEGY.SILENT then
        return _("Auto")
    else
        return _("Disable")
    end
end

function Bookhoard:toggleAutoSync()
    if not self.settings.auto_sync
        and Device:hasSeamlessWifiToggle()
        and G_reader_settings:readSetting("wifi_enable_action") ~= "turn_on" then
        UIManager:show(InfoMessage:new{
            text = _("Set 'Action when Wi-Fi is off' to 'turn on' in Network settings to enable auto sync."),
        })
        return
    end
    self.settings.auto_sync = not self.settings.auto_sync
    self:registerEvents()
    G_reader_settings:saveSetting(self.settings_key, self.settings)

    if self.settings.auto_sync and self.ui.doc_settings then
        self:getProgress(true, true)
    end
end

function Bookhoard:setupMenuOrder()
    local settings_dir = DataStorage:getSettingsDir()
    self:patchMenuOrderFile(settings_dir .. "/reader_menu_order.lua")
    self:patchMenuOrderFile(settings_dir .. "/filemanager_menu_order.lua")
end

function Bookhoard:patchMenuOrderFile(filepath)
    local default_tools = {
        "read_timer",
        "calibre",
        "bookhoard_sync",
        "exporter",
        "statistics",
        "progress_sync",
        "move_to_archive",
        "wallabag",
        "news_downloader",
        "text_editor",
        "profiles",
        "qrclipboard",
        "----------------------------",
        "more_tools",
    }

    local attr = lfs.attributes(filepath)
    if not attr then
        local f = io.open(filepath, "w")
        if not f then return end
        f:write("return {\n    tools = {\n")
        for _, id in ipairs(default_tools) do
            f:write('        "' .. id .. '",\n')
        end
        f:write("    },\n}\n")
        f:close()
        logger.dbg("Bookhoard: created menu order file", filepath)
        return
    end

    local existing = dofile(filepath)
    if not existing or type(existing) ~= "table" then return end

    local tools = existing.tools
    if not tools or type(tools) ~= "table" then return end

    for _, id in ipairs(tools) do
        if id == "bookhoard" then return end
    end

    local insert_pos = nil
    for i, id in ipairs(tools) do
        if id == "calibre" then
            insert_pos = i + 1
            break
        end
    end
    if not insert_pos then
        insert_pos = 2
    end

    table.insert(tools, insert_pos, "bookhoard")
    existing.tools = tools

    local f = io.open(filepath, "w")
    if not f then return end
    f:write("return {\n")
    for key, val in pairs(existing) do
        if type(val) == "table" then
            f:write("    " .. key .. " = {\n")
            for _, id in ipairs(val) do
                f:write('        "' .. id .. '",\n')
            end
            f:write("    },\n")
        end
    end
    f:write("}\n")
    f:close()
    logger.dbg("Bookhoard: patched menu order file", filepath)
end

function Bookhoard:startRegistration()
    if not self.settings.server_url or self.settings.server_url == "" then
        UIManager:show(InfoMessage:new{
            text = _("Please set your server URL first."),
            timeout = 3,
        })
        return
    end

    if NetworkMgr:willRerunWhenOnline(function() self:startRegistration() end) then
        return
    end

    local device_name = Device.model or "KOReader Device"
    local device_identifier = Device:info() or device_name

    UIManager:show(InfoMessage:new{
        text = _("Registering device…"),
        timeout = 1,
    })

    UIManager:scheduleIn(0.5, function()
        local api = BookhoardAPI:new({ server_url = self.settings.server_url })
        local ok, result = api:registerDevice(device_name, device_identifier)

        if not ok then
            UIManager:show(InfoMessage:new{
                text = T(_("Registration failed: %1"),
                    result and result.error or _("unknown error")),
            })
            return
        end

        self.registration_id = result.registration_id

        self.waiting_dialog = InfoMessage:new{
            text = T(_("Device registered on server.\n\nOpen your Bookhoard web UI and go to:\n%1/devices\n\nApprove this device in the \"Pending Device Registrations\" section.\n\nWaiting for approval…"), self.settings.server_url),
        }
        UIManager:show(self.waiting_dialog)

        self:startRegistrationPoll()
    end)
end

function Bookhoard:startRegistrationPoll()
    if self.registration_poll_scheduled then return end
    self.registration_poll_scheduled = true

    local function poll()
        self.registration_poll_scheduled = false

        if not self.registration_id then return end

        local api = BookhoardAPI:new({ server_url = self.settings.server_url })
        local ok, result = api:checkRegistrationStatus(self.registration_id)

        if not ok then
            if result and result.status == 410 then
                self.registration_id = nil
                UIManager:show(InfoMessage:new{
                    text = _("Registration expired. Please try again."),
                })
                return
            end
            self.registration_poll_scheduled = true
            UIManager:scheduleIn(3, poll)
            return
        end

        if result.status == "approved" then
            self.registration_id = nil
            self.settings.auth_token = result.auth_token
            self.settings.device_id = result.device_id and tostring(result.device_id) or nil
            G_reader_settings:saveSetting(self.settings_key, self.settings)
            self:registerEvents()
            self:setupOPDS()
            if self.waiting_dialog then
                UIManager:close(self.waiting_dialog)
                self.waiting_dialog = nil
            end
            UIManager:show(InfoMessage:new{
                text = _("Device registered successfully!"),
                timeout = 3,
            })
        elseif result.status == "pending" then
            self.registration_poll_scheduled = true
            UIManager:scheduleIn(3, poll)
        else
            self.registration_id = nil
            UIManager:show(InfoMessage:new{
                text = T(_("Registration %1"), result.status or _("failed")),
            })
        end
    end

    UIManager:scheduleIn(3, poll)
end

function Bookhoard:setupOPDS()
    if not self.settings.server_url or not self.settings.device_id
        or not self.settings.auth_token then
        return false
    end

    local opds_base_url = self.settings.server_url
        .. "/opds/devices/" .. self.settings.device_id .. "/catalog"
    local opds_url = opds_base_url .. "?token=" .. self.settings.auth_token

    local opds_settings_file = DataStorage:getSettingsDir() .. "/opds.lua"
    local opds_settings = LuaSettings:open(opds_settings_file)
    local servers = opds_settings:readSetting("servers", {})

    local found = false
    for i, server in ipairs(servers) do
        if server.url and (
            server.url == opds_url
            or server.url == opds_base_url
            or server.url:sub(1, #opds_base_url + 1) == opds_base_url .. "?"
        ) then
            servers[i] = {
                title = "Bookhoard",
                url = opds_url,
            }
            found = true
            break
        end
    end

    if not found then
        table.insert(servers, {
            title = "Bookhoard",
            url = opds_url,
        })
    end

    opds_settings:saveSetting("servers", servers)
    opds_settings:flush()

    if self.ui.opds then
        self.ui.opds.servers = servers
    end

    return true
end

function Bookhoard:getLastPercent()
    if self.ui.document.info.has_pages then
        return Math.roundPercent(self.ui.paging:getLastPercent())
    else
        return Math.roundPercent(self.ui.rolling:getLastPercent())
    end
end

function Bookhoard:getLastProgress()
    if self.ui.document.info.has_pages then
        return self.ui.paging:getLastProgress()
    else
        return self.ui.rolling:getLastProgress()
    end
end

function Bookhoard:getFileSHA256()
    local cached = self.ui.doc_settings:readSetting("bookhoard_sha256")
    if cached then return cached end

    local file = io.open(self.ui.document.file, "rb")
    if not file then return nil end
    local data = file:read("*a")
    file:close()

    local hash = sha256hex(data)
    if hash then
        self.ui.doc_settings:saveSetting("bookhoard_sha256", hash)
    end
    return hash
end

function Bookhoard:getBookhoardUUID()
    if not self.ui.doc_settings then return nil end
    return self.ui.doc_settings:readSetting("bookhoard_uuid")
end

function Bookhoard:getContextText()
    if not self.ui.document or self.ui.document.info.has_pages then
        return ""
    end
    local xp = self:getLastProgress()
    if not xp then return "" end

    local text = self.ui.document:getTextFromXPointer(xp)
    if not text or text == "" then
        text = self.ui.document:getTextFromXPointer(self.ui.document:getNormalizedXPointer(xp))
    end
    if not text or text == "" then return "" end

    local char_offset = tonumber(xp:match("text%(%)%.?(%d+)")) or 0
    if char_offset > 0 and char_offset < #text then
        text = text:sub(char_offset + 1)
    end

    if #text > 100 then
        text = text:sub(1, 100)
    end

    text = text:gsub("%s+", " "):match("^%s*(.-)%s*$") or ""
    return text
end

function Bookhoard:collectBookData()
    local props = self.ui.doc_props
    local file_path = self.ui.document.file

    local title = props.display_title or ""
    local authors = {}
    if props.authors then
        authors = { props.authors }
    end
    json_util.InitArray(authors)

    local file_sha256 = self:getFileSHA256()
    local book_uuid = self:getBookhoardUUID()

    local percentage = self:getLastPercent()
    local progress = self:getLastProgress()
    local page = self.ui:getCurrentPage()
    local total_pages = self.ui.document:getPageCount()
    local context_text = self:getContextText()

    local book_data = {
        uuid = book_uuid,
        sha256 = file_sha256,
        title = title,
        authors = authors,
        percentage = percentage,
        context_text = context_text,
        page = page,
        total_pages = total_pages,
        file_path = file_path,
        device_info = {
            koreader_version = require("version"):getCurrentRevision(),
            device_model = Device.model,
        },
    }

    -- epubcfi (a CREngine xpointer) is only meaningful for reflowable (rolling)
    -- documents. Paging docs (PDF/comics/DjVu) carry their position in
    -- page/total_pages; a bare page number here would be a JSON number into a
    -- server *string field and fail the bind. Reflowable EPUBs are always
    -- rolling, so they keep sending the xpointer exactly as before.
    if not self.ui.document.info.has_pages then
        book_data.epubcfi = progress
    end

    return book_data
end

-- KOReader stores annotations in one of two models depending on version:
--  v2 (2024.07+): self.ui.annotation.annotations; entries carry
--     text = highlighted text, note = user note, drawer/color style,
--     pos0/pos1 = xpointer (CRE) or {page=N} table (pdf), page = xpointer
--     (CRE) or number (pdf). Sidecar key "annotations".
--  v1 (older):    self.ui.bookmark.bookmarks; entries carry text = note
--     label / bookmark title, notes = highlighted text, pos0/pos1 strings.
-- Returns "v2"/"v1"/nil and the entries array.
function Bookhoard:getAnnotationStore()
    if self.ui.annotation and type(self.ui.annotation.annotations) == "table" then
        return "v2", self.ui.annotation.annotations
    end
    if self.ui.bookmark and type(self.ui.bookmark.bookmarks) == "table" then
        return "v1", self.ui.bookmark.bookmarks
    end
    return nil, nil
end

function Bookhoard:collectAnnotations()
    local bookmarks = json_util.InitArray({})
    local highlights = json_util.InitArray({})
    local notes = json_util.InitArray({})

    local model, entries = self:getAnnotationStore()
    if not model then
        return bookmarks, highlights, notes
    end

    local file_sha256 = self:getFileSHA256()
    local total_pages = self.ui.document:getPageCount()
    local has_pages = self.ui.document.info.has_pages

    local function wirePos(p, fallback_page)
        -- v2 pdf positions are tables {page=N,x,y}; the wire format is a
        -- string — the page number is the durable locator there.
        if type(p) == "table" then
            return tostring(p.page or fallback_page or "")
        end
        return p or ""
    end

    local function echoColor(bm)
        -- Color the device can vouch for as user intent:
        --  - entries we applied ourselves carry the device DEFAULT color
        --    (set at apply time); pushing it back would clobber the web
        --    color. Suppress it unless the user has edited the entry since
        --    (datetime_updated — set by KOReader on any modification).
        --  - device-native entries carry a user-chosen (or default) color;
        --    those push their color as before.
        if bm.bookhoard_dedup_key and not bm.datetime_updated then
            return nil
        end
        return bm.color
    end

    for _, bm in ipairs(entries) do
        local sel_text, user_note
        if model == "v2" then
            if bm.drawer then
                sel_text = bm.text or ""  -- highlighted text
                user_note = bm.note or "" -- user note
            else
                -- Page bookmark. `drawer` is the only reliable highlight
                -- discriminator: KOReader auto-fills text = "in Chapter X"
                -- on bookmarks (updateItemByXPointer), so text-presence
                -- would misclassify every bookmark as a highlight on echo.
                -- The user label lives in `note`.
                sel_text = ""
                user_note = bm.note or ""
            end
        else
            sel_text = bm.notes or ""   -- v1: notes held the highlighted text
            user_note = bm.text or ""   -- v1: text held the note/label
        end

        -- Thin-client policy: no per-annotation CRE lookups here (a
        -- getPageFromXPointer call each is the most expensive thing this
        -- loop can do on weak hardware). Percentages are arithmetic for
        -- paging documents and simply omitted for CRE documents — the
        -- server derives them from the locator against the actual book.
        local page_num = tonumber(bm.page) or 0
        local percentage = has_pages and total_pages > 0 and (page_num / total_pages) or nil

        local chapter = bm.chapter or ""
        if type(chapter) == "table" then chapter = "" end

        local entry = {
            chapter = chapter,
            datetime = bm.datetime or "",
            notes = user_note,
            pos0 = wirePos(bm.pos0, bm.page),
            pos1 = wirePos(bm.pos1, bm.page),
            page = tostring(bm.page or ""),
            text = sel_text,
            book_sha256 = file_sha256,
        }
        if percentage then
            entry.percentage = Math.roundPercent(percentage)
        end
        if bm.bookhoard_dedup_key then
            -- Echo identity for entries received from the server: lets the
            -- server match this push to the original row instead of
            -- minting a duplicate (device locators ≠ web locators, so the
            -- computed key would never match).
            entry.dedup_key = bm.bookhoard_dedup_key
        end

        local has_text = sel_text ~= ""
        local has_notes = user_note ~= ""

        if has_text and has_notes then
            entry.type = "note"
            table.insert(notes, entry)
        elseif has_text then
            entry.type = "highlight"
            local ec = echoColor(bm)
            if ec then
                entry.color = ec
            end
            table.insert(highlights, entry)
        else
            entry.type = "bookmark"
            entry.text = user_note -- bookmark label travels in `text`
            table.insert(bookmarks, entry)
        end
    end

    return bookmarks, highlights, notes
end

function Bookhoard:syncToProgress(progress, percentage)
    logger.dbg("Bookhoard: sync to progress", progress, percentage)
    if self.ui.document.info.has_pages then
        local page = tonumber(progress)
        if page then
            self.ui:handleEvent(Event:new("GotoPage", page))
        end
    elseif progress and progress:match("^/body/") then
        self.ui:handleEvent(Event:new("GotoXPointer", progress))
    elseif percentage then
        -- Navigation goes through events (ReaderRolling/ReaderPaging both
        -- implement onGotoPercent and refresh the view themselves). Calling
        -- document:gotoPercent directly crashed KOReader — no such method
        -- exists on documents.
        self.ui:handleEvent(Event:new("GotoPercent", percentage * 100))
    end
end

function Bookhoard:updateProgress(ensure_networking, interactive)
    if not self:isConfigured() then
        if interactive then
            UIManager:show(InfoMessage:new{
                text = _("Please configure and register your device first."),
                timeout = 3,
            })
        end
        return
    end

    if not self.settings.sync_progress then return end

    if not self.ui.document then
        if interactive then
            UIManager:show(InfoMessage:new{
                text = _("No document open."),
                timeout = 3,
            })
        end
        return
    end

    local now = UIManager:getElapsedTimeSinceBoot()
    if not interactive and now - self.push_timestamp <= API_CALL_DEBOUNCE_DELAY then
        return
    end

    if ensure_networking
        and NetworkMgr:willRerunWhenOnline(function() self:updateProgress(ensure_networking, interactive) end) then
        return
    end

    UIManager:scheduleIn(0.5, function()
        self:_doUpdateProgress(interactive)
    end)

    self.push_timestamp = now
end

function Bookhoard:_doUpdateProgress(interactive)
    local book_data = self:collectBookData()
    if not book_data then return end

    if self.settings.sync_bookmarks or self.settings.sync_highlights or self.settings.sync_notes then
        local bm, hl, nt = self:collectAnnotations()
        book_data.bookmarks = self.settings.sync_bookmarks and bm or {}
        book_data.highlights = self.settings.sync_highlights and hl or {}
        book_data.notes = self.settings.sync_notes and nt or {}
    else
        book_data.bookmarks = {}
        book_data.highlights = {}
        book_data.notes = {}
    end

    local api = self:getAPI()
    local ok, result = api:syncProgress(book_data, self.settings.sync_mode)

    UIManager:nextTick(function()
        if ok then
            logger.dbg("Bookhoard: progress pushed successfully")
            if result and result.book_results then
                for _, br in ipairs(result.book_results) do
                    if br.synced and br.book_uuid and br.sha256 == book_data.sha256 then
                        self.ui.doc_settings:saveSetting("bookhoard_uuid", br.book_uuid)
                        self.ui.doc_settings:flush()
                        break
                    end
                end
            end
            if interactive then
                UIManager:show(InfoMessage:new{
                    text = _("Progress has been pushed."),
                    timeout = 3,
                })
            end
        else
            logger.warn("Bookhoard: failed to push progress")
            if interactive then
                UIManager:show(InfoMessage:new{
                    text = _("Failed to push progress. Check your network connection."),
                    timeout = 3,
                })
            end
        end
    end)
end

function Bookhoard:getProgress(ensure_networking, interactive)
    if not self:isConfigured() then
        if interactive then
            UIManager:show(InfoMessage:new{
                text = _("Please configure and register your device first."),
                timeout = 3,
            })
        end
        return
    end

    if not self.settings.sync_progress then return end

    local now = UIManager:getElapsedTimeSinceBoot()
    if not interactive and now - self.pull_timestamp <= API_CALL_DEBOUNCE_DELAY then
        return
    end

    if ensure_networking
        and NetworkMgr:willRerunWhenOnline(function() self:getProgress(ensure_networking, interactive) end) then
        return
    end

    local book_uuid = self:getBookhoardUUID()
    if not book_uuid then
        -- No cached UUID yet (first open of a freshly downloaded book). Bootstrap
        -- it by pushing once: the server resolves the book by SHA-256 and returns
        -- the UUID, which the push handler caches. Then retry the pull.
        if interactive then
            UIManager:show(InfoMessage:new{
                text = _("Linking this book to Bookhoard first…"),
                timeout = 3,
            })
        end
        self:_bootstrapUUIDThenPull(interactive)
        return
    end

    UIManager:scheduleIn(0.5, function()
        self:_doGetProgress(interactive)
    end)

    self.pull_timestamp = now
end

-- Link the current document to Bookhoard WITHOUT pushing progress, then run
-- the pull. The UUID is learned via a read-only SHA-256 resolve; only when
-- the server does not know the hash at all (sideloaded book) do we fall back
-- to the legacy push-bootstrap, and only for background (non-interactive)
-- flows — a user-initiated pull must never push this device's position.
function Bookhoard:_linkBookThenPull(interactive)
    local file_sha256 = self:getFileSHA256()
    if not file_sha256 then
        if interactive then
            UIManager:show(InfoMessage:new{
                text = _("Could not read this file to link it with Bookhoard."),
                timeout = 3,
            })
        end
        return
    end

    UIManager:scheduleIn(0.5, function()
        local api = self:getAPI()
        local ok, result = api:resolveBook(file_sha256)

        UIManager:nextTick(function()
            if not self.ui.doc_settings then return end

            if ok and type(result) == "table" and result.book_uuid then
                self.ui.doc_settings:saveSetting("bookhoard_uuid", result.book_uuid)
                self.ui.doc_settings:flush()
                logger.dbg("Bookhoard: linked book by SHA-256 to", result.book_uuid)
                self:_doGetProgress(interactive)
                return
            end

            if result and result.status == 404 then
                -- Genuinely unknown to the server (sideloaded book). The
                -- legacy push-bootstrap only ever linked such books via fuzzy
                -- title/author matching; keep that as a background attempt,
                -- but tell interactive users to push explicitly.
                if interactive then
                    UIManager:show(InfoMessage:new{
                        text = _("This book is not in your Bookhoard library yet. Push progress to link it."),
                        timeout = 4,
                    })
                else
                    self:_bootstrapUUIDThenPull(interactive)
                end
                return
            end

            logger.warn("Bookhoard: failed to resolve book by SHA-256")
            if interactive then
                UIManager:show(InfoMessage:new{
                    text = _("Could not reach Bookhoard. Check your network connection and try again."),
                    timeout = 4,
                })
            end
        end)
    end)
end

-- Legacy fallback: establish the bookhoard UUID for the current document by
-- pushing once, then (once cached) perform the originally-requested pull.
-- Only reached from background flows when the SHA-256 resolve reported the
-- book as unknown to the server; the push mainly serves fuzzy title/author
-- linking and device alias registration for sideloaded books.
function Bookhoard:_bootstrapUUIDThenPull(interactive)
    self:updateProgress(false, false)
    UIManager:scheduleIn(2, function()
        if self:getBookhoardUUID() then
            self:_doGetProgress(interactive)
        elseif interactive then
            UIManager:show(InfoMessage:new{
                text = _("Could not link this book. Check your network and try again."),
                timeout = 3,
            })
        end
    end)
end

function Bookhoard:_doGetProgress(interactive)
    local book_uuid = self:getBookhoardUUID()
    if not book_uuid then return end

    local api = self:getAPI()
    local ok, result = api:getMetadata(book_uuid)

    UIManager:nextTick(function()
        if not ok or not result then
            if interactive then
                UIManager:show(InfoMessage:new{
                    text = _("Failed to pull progress."),
                    timeout = 3,
                })
            end
            return
        end

        if not result.progress then
            if interactive then
                UIManager:show(InfoMessage:new{
                    text = _("No progress found for this document."),
                    timeout = 3,
                })
            end
            return
        end

        if result.annotations then
            self:applyServerAnnotations(result.annotations)
        end

        local progress = result.progress
        local percentage = self:getLastPercent()
        local server_percentage = progress.percentage or 0

        if percentage == server_percentage then
            if interactive then
                UIManager:show(InfoMessage:new{
                    text = _("Progress is already synchronized."),
                    timeout = 3,
                })
            end
            return
        end

        local self_older = server_percentage > percentage

        local nav_target
        local sync_text
        if self.ui.document.info.has_pages then
            -- Fixed-layout: the page index is the canonical locator. CFI/xpointer
            -- are meaningless for image-based content, so use progress.page directly.
            local total = progress.total_pages or self.ui.document:getPageCount()
            local target_page = progress.page
                or math.min(Math.round(server_percentage * total), total)
            nav_target = progress.page
            sync_text = T(_("Sync to page %1 of %2 from server?"), target_page, total)
        else
            nav_target = progress.koreader_xpointer or progress.epubcfi or progress.page
            local total = self.ui.document:getPageCount()
            local target_page = math.min(Math.round(server_percentage * total), total)
            sync_text = T(_("Sync to page %1 of %2 from server?"), target_page, total)
        end

        if self_older then
            if self.settings.sync_forward == SYNC_STRATEGY.SILENT then
                self:syncToProgress(nav_target, server_percentage)
                self:_showSyncedMessage()
            elseif self.settings.sync_forward == SYNC_STRATEGY.PROMPT then
                UIManager:show(ConfirmBox:new{
                    text = sync_text,
                    ok_callback = function()
                        self:syncToProgress(nav_target, server_percentage)
                    end,
                })
            end
        else
            if self.settings.sync_backward == SYNC_STRATEGY.SILENT then
                self:syncToProgress(nav_target, server_percentage)
                self:_showSyncedMessage()
            elseif self.settings.sync_backward == SYNC_STRATEGY.PROMPT then
                UIManager:show(ConfirmBox:new{
                    text = sync_text,
                    ok_callback = function()
                        self:syncToProgress(nav_target, server_percentage)
                    end,
                })
            end
        end
    end)
end

function Bookhoard:applyServerAnnotations(annotations)
    if not annotations then return end
    local model, entries = self:getAnnotationStore()
    if not model then return end

    local changed = false
    local notify_events = {}
    local function notify(ev)
        notify_events[#notify_events + 1] = ev
    end
    local has_pages = self.ui.document.info.has_pages

    local function findLocal(server_entry)
        -- Identity match by dedup key: survives pos0 drift (improved
        -- server conversion) and, crucially, never cross-matches a
        -- DIFFERENT annotation that merely shares the position.
        local key = server_entry.dedup_key
        if key and key ~= "" then
            for i, bm in ipairs(entries) do
                if bm.bookhoard_dedup_key == key then
                    return i
                end
            end
            return nil -- keyed but not present: a real new entry
        end
        -- Legacy serve (no key): fall back to pos0 matching.
        local pos0 = server_entry.pos0 or ""
        if pos0 == "" then return nil end
        for i, bm in ipairs(entries) do
            local p = bm.pos0
            if type(p) == "table" then p = tostring(p.page or "") end
            if p == pos0 then
                return i
            end
        end
        return nil
    end

    -- Get/set the highlighted text and the user note with model-appropriate
    -- field names (v2: text/note; v1: notes/text).
    local function getFields(bm)
        if model == "v2" then
            return bm.text or "", bm.note or ""
        end
        return bm.notes or "", bm.text or ""
    end
    local function setFields(bm, sel_text, user_note)
        if model == "v2" then
            bm.text = sel_text ~= "" and sel_text or nil
            bm.note = user_note ~= "" and user_note or nil
        else
            bm.notes = sel_text
            bm.text = user_note
        end
    end

    local function isValidPos0(pos0)
        -- Rolling (CRE) documents address bookmarks/highlights by xpointer;
        -- paging documents (PDF/comics/DjVu) by page number. Anything else
        -- (a raw "cfi:" locator, an epubcfi(...) string, a JSON anchor, …)
        -- cannot be placed in this document and must not become a local
        -- bookmark: it could never be matched again and would be re-pushed
        -- to the server as a junk duplicate on the next sync.
        if pos0 == "" then return false end
        if has_pages then
            return tonumber(pos0) ~= nil
        end
        return pos0:sub(1, 6) == "/body/"
    end

    local function makePageFromPos(pos0)
        if has_pages then
            local page = tonumber(pos0:match("(%d+)$"))
            return page or pos0
        end
        return pos0
    end

    -- v2 pdf positions are {page=N} tables; CRE keeps xpointer strings.
    local function makeLocalPos(pos)
        if has_pages then
            local page = tonumber(pos)
            return page and { page = page } or nil
        end
        return pos
    end

    local function addOrUpdate(server_entry, has_text)
        local pos0 = server_entry.pos0 or ""
        if not isValidPos0(pos0) then return end
        local srv_text = server_entry.text or ""
        local srv_notes = server_entry.notes or ""
        local idx = findLocal(server_entry)
        if idx then
            local bm = entries[idx]
            local cur_text, cur_note = getFields(bm)
            if cur_text ~= srv_text or cur_note ~= srv_notes then
                setFields(bm, srv_text, srv_notes)
                if server_entry.color and server_entry.color ~= "" then bm.color = server_entry.color end
                notify({ bm })
                changed = true
            end
        elseif model == "v2" then
            -- `drawer` marks a renderable highlight in v2; its absence
            -- makes the entry a page bookmark (label goes in `note`).
            local entry = {
                datetime = server_entry.datetime ~= "" and server_entry.datetime or nil,
                chapter = server_entry.chapter ~= "" and server_entry.chapter or nil,
                pos0 = makeLocalPos(pos0),
                pos1 = makeLocalPos(server_entry.pos1 ~= "" and server_entry.pos1 or pos0),
                page = makePageFromPos(pos0),
            }
            if server_entry.dedup_key and server_entry.dedup_key ~= "" then
                entry.bookhoard_dedup_key = server_entry.dedup_key
            end
            if has_text then
                entry.drawer = "lighten"
                entry.text = srv_text
                if srv_notes ~= "" then entry.note = srv_notes end
                -- Color: the server maps the web color into KOReader's
                -- named palette — use it when it arrives as a plain name so
                -- synced highlights keep their web color. Anything else
                -- (absent, or a raw hex that slipped through) falls back to
                -- the device default (view.highlight.saved_color, "yellow"
                -- on color screens — nil would render grey via darkenRect).
                -- Echoes of applied entries never push this color back
                -- (collectAnnotations) unless the user edited them, so the
                -- stored web color never drifts.
                local srv_color = server_entry.color or ""
                if srv_color ~= "" and not srv_color:find("^#") then
                    entry.color = srv_color
                else
                    entry.color = (self.ui.view and self.ui.view.highlight
                        and self.ui.view.highlight.saved_color) or "yellow"
                end
            else
                if srv_text ~= "" then entry.note = srv_text end -- bookmark label
            end
            local index = self.ui.annotation:addItem(entry)
            notify({ entry, index_modified = index })
            changed = true
        else
            local entry = {
                page = makePageFromPos(pos0),
                pos0 = pos0,
                pos1 = server_entry.pos1 or pos0,
                datetime = server_entry.datetime or "",
                notes = srv_text,
                text = srv_notes,
            }
            if has_text then
                local srv_color = server_entry.color or ""
                if srv_color ~= "" and not srv_color:find("^#") then
                    entry.color = srv_color
                end
            end
            if server_entry.dedup_key and server_entry.dedup_key ~= "" then
                entry.bookhoard_dedup_key = server_entry.dedup_key
            end
            if server_entry.chapter and server_entry.chapter ~= "" then entry.chapter = server_entry.chapter end
            table.insert(entries, entry)
            changed = true
        end
    end

    if self.settings.sync_highlights and annotations.highlights then
        for _, hl in ipairs(annotations.highlights) do
            addOrUpdate(hl, true)
        end
    end

    if self.settings.sync_notes and annotations.notes then
        for _, note in ipairs(annotations.notes) do
            addOrUpdate(note, true)
        end
    end

    if self.settings.sync_bookmarks and annotations.bookmarks then
        for _, bm in ipairs(annotations.bookmarks) do
            addOrUpdate(bm, false)
        end
    end

    local function findLocalByTombstone(del, want_highlight)
        local key = del.dedup_key
        if key and key ~= "" then
            for i, bm in ipairs(entries) do
                if bm.bookhoard_dedup_key == key then
                    return i
                end
            end
            return nil -- keyed tombstone: only its own entry may be removed
        end
        -- Legacy tombstone (no key): pos0 match, restricted to the same
        -- annotation kind so a highlight tombstone never eats a bookmark
        -- that shares the position.
        local pos0 = del.pos0 or ""
        if pos0 == "" then return nil end
        for i, bm in ipairs(entries) do
            local p = bm.pos0
            if type(p) == "table" then p = tostring(p.page or "") end
            if p == pos0 and (not not bm.drawer) == want_highlight then
                return i
            end
        end
        return nil
    end

    if annotations.deleted_highlights then
        for _, del in ipairs(annotations.deleted_highlights) do
            local idx = findLocalByTombstone(del, true)
            local bm = idx and entries[idx]
            local cur_text = bm and (getFields(bm)) or ""
            if idx and cur_text ~= "" then
                table.remove(entries, idx)
                notify({ bm, index_modified = -idx })
                changed = true
            end
        end
    end

    if annotations.deleted_bookmarks then
        for _, del in ipairs(annotations.deleted_bookmarks) do
            local idx = findLocalByTombstone(del, false)
            if idx then
                local removed = entries[idx]
                table.remove(entries, idx)
                notify({ removed, index_modified = -idx })
                changed = true
            end
        end
    end

    if changed then
        -- addItem() inserts v2 entries at their sorted position already;
        -- explicit re-sorting is a v1-model need only.
        if model == "v1" and self.ui.bookmark and self.ui.bookmark.onSortBookmarks then
            self.ui.bookmark:onSortBookmarks()
        end
        if self.ui.saveSettings then
            self.ui:saveSettings()
        end
        -- ReaderView caches each rendered page's highlight boxes and only
        -- invalidates them on AnnotationsModified — without this, applied
        -- annotations don't paint until restart despite the repaint.
        -- Payloads MUST be tables: ReaderThumbnail iterates them, and a
        -- nil payload crashed KOReader (#nil) on pull. Per-item payloads
        -- with index_modified mirror the native add/update/remove
        -- dispatches exactly (page-level cache invalidation + cached box
        -- index shifting).
        for _, ev in ipairs(notify_events) do
            self.ui:handleEvent(Event:new("AnnotationsModified", ev))
        end
        if self.ui.dialog then
            UIManager:setDirty(self.ui.dialog, "ui")
        end
        logger.dbg("Bookhoard: annotations updated from server")
    end
end

function Bookhoard:_showSyncedMessage()
    UIManager:show(InfoMessage:new{
        text = _("Progress has been synchronized."),
        timeout = 3,
    })
end

function Bookhoard:_onCloseDocument()
    self.onResume = nil
    self.onSuspend = nil
    NetworkMgr:goOnlineToRun(function()
        self:updateProgress(false, false)
    end)
end

function Bookhoard:schedulePeriodicPush()
    UIManager:unschedule(self.periodic_push_task)
    UIManager:scheduleIn(PERIODIC_PUSH_DELAY, self.periodic_push_task)
    self.periodic_push_scheduled = true
end

function Bookhoard:_onPageUpdate(page)
    if page == nil then return end

    if self.last_page ~= page then
        self.last_page = page
        self.page_update_counter = self.page_update_counter + 1
        if self.periodic_push_scheduled
            or (self.settings.pages_before_update
                and self.page_update_counter >= self.settings.pages_before_update) then
            self:schedulePeriodicPush()
        end
    end
end

function Bookhoard:_onResume()
    if Device:hasWifiRestore() and NetworkMgr.wifi_was_on
        and G_reader_settings:isTrue("auto_restore_wifi") then
        return
    end
    UIManager:scheduleIn(1, function()
        self:getProgress(true, false)
    end)
end

function Bookhoard:_onSuspend()
    self:updateProgress(true, false)
end

function Bookhoard:_onNetworkConnected()
    UIManager:scheduleIn(0.5, function()
        self:getProgress(false, false)
    end)
end

function Bookhoard:_onNetworkDisconnecting()
    self:updateProgress(false, false)
end

function Bookhoard:onCloseWidget()
    UIManager:unschedule(self.periodic_push_task)
    self.periodic_push_task = nil
end

return Bookhoard