From bec56ba8147302c19b1f9592bc7898fee03cb634 Mon Sep 17 00:00:00 2001 From: Kaj Schittecat Date: Tue, 13 Oct 2026 11:48:26 +0200 Subject: [PATCH] reader: page through big text files, open card documents from Files, and a tool to fill a card Asked for on Discord (prepper use: books and documentation offline on the card). - The Reader opens a .txt or .md file on the card as a 16 KB window at an offset carried in the URL (sd:/x.txt?o=49152), cut back to a line (or a whole UTF-8 character), with Next page / Previous page / Back to the start links and the position in percent. A book of any size pages through; Back works as usual. - Files opens HTML on the card in the Reader (its links work there, so a folder of pages browses offline) and text or markdown too big for the 8 KB editor; small text files keep the editor. - scripts/sdpack.py (standard library only): an EPUB becomes /library// with one page per chapter (long chapters split under 30 KB, linked in order) and a contents page; .txt/.md are copied as they are; /home.htm lists everything, so the Reader, which opens sd:/home.htm, starts on the library. Tested with a generated EPUB. ZIM (Wikipedia) files are not read on the device: their clusters are zstd/xz blobs of 1-2 MB and the files run to tens of GB on FAT32; converting on a computer keeps the device side small. Co-Authored-By: Claude Opus 5.5 --- scripts/sdpack.py | 174 ++++++++++++++++++++++++++++++++++++++++ src/ui-touch/UITask.cpp | 105 +++++++++++++++++++++++- 2 files changed, 275 insertions(+), 4 deletions(-) create mode 100755 scripts/sdpack.py diff --git a/scripts/sdpack.py b/scripts/sdpack.py new file mode 100755 index 0000000..5c79471 --- /dev/null +++ b/scripts/sdpack.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""Put books and documents on a wadamesh SD card for the on-device Reader. + +The Reader opens sd:/home.htm when it starts, follows links between pages on the +card without Wi-Fi, and pages through large .txt / .md files on its own. This +tool fills a card for it: + + python3 scripts/sdpack.py /Volumes/SDCARD book.epub notes.txt manual.md + +- An EPUB becomes /library// : one small page per chapter (split further when + a chapter is long, each part linked to the next) plus index.htm, its contents. +- A .txt or .md file is copied to /library/ as it is: the Reader pages through it. +- /home.htm is (re)written to list everything in /library, so the Reader opens on it. + +Only the Python standard library. The card must be FAT32 (the firmware does not +read exFAT). Pages stay well under the Reader's limits (it reads up to 48 KB of a +page on the smallest boards and shows up to 280 links). +""" +import html, os, posixpath, re, shutil, sys, zipfile +from html.parser import HTMLParser + +PART_MAX = 30 * 1024 # bytes of text per page: under the Reader's smallest buffer +LIB = "library" + + +def slug(name): + s = re.sub(r"[^A-Za-z0-9]+", "-", name).strip("-").lower() + return (s or "book")[:40] + + +class _Text(HTMLParser): + """XHTML chapter -> blocks of (kind, text): headings and paragraphs, nothing else.""" + BLOCK = {"p", "div", "li", "br", "tr", "blockquote", "pre"} + HEAD = {"h1", "h2", "h3", "h4", "h5", "h6"} + SKIP = {"script", "style", "head", "title"} + + def __init__(self): + super().__init__(convert_charrefs=True) + self.blocks, self.cur, self.kind, self.skip = [], [], "p", 0 + + def _flush(self): + t = re.sub(r"\s+", " ", "".join(self.cur)).strip() + if t: + self.blocks.append((self.kind, t)) + self.cur, self.kind = [], "p" + + def handle_starttag(self, tag, attrs): + if tag in self.SKIP: + self.skip += 1 + elif tag in self.HEAD: + self._flush(); self.kind = "h" + elif tag in self.BLOCK: + self._flush() + + def handle_endtag(self, tag): + if tag in self.SKIP: + self.skip = max(0, self.skip - 1) + elif tag in self.HEAD or tag in self.BLOCK: + self._flush() + + def handle_data(self, data): + if not self.skip: + self.cur.append(data) + + def close(self): + super().close(); self._flush() + return self.blocks + + +def _page(title, body, nav): + return ("%s\n" + "

%s

\n%s\n

%s

\n\n") % ( + html.escape(title), html.escape(title), body, " | ".join(nav)) + + +def _opf(z): + container = z.read("META-INF/container.xml").decode("utf-8", "replace") + opf_path = re.search(r'full-path="([^"]+)"', container).group(1) + opf = z.read(opf_path).decode("utf-8", "replace") + base = posixpath.dirname(opf_path) + title = re.search(r"]*>(.*?)", opf, re.S) + items = {m.group(1): m.group(2) for m in + re.finditer(r']*?id="([^"]+)"[^>]*?href="([^"]+)"', opf)} + items.update({m.group(2): m.group(1) for m in + re.finditer(r']*?href="([^"]+)"[^>]*?id="([^"]+)"', opf)}) + spine = re.findall(r']*?idref="([^"]+)"', opf) + files = [posixpath.normpath(posixpath.join(base, items[i])) for i in spine if i in items] + return (html.unescape(title.group(1).strip()) if title else None), files + + +def pack_epub(src, card): + z = zipfile.ZipFile(src) + title, files = _opf(z) + title = title or os.path.splitext(os.path.basename(src))[0] + d = os.path.join(card, LIB, slug(title)) + os.makedirs(d, exist_ok=True) + pages = [] # (file name, page title) + for idx, f in enumerate(files): + try: + raw = z.read(f).decode("utf-8", "replace") + except KeyError: + continue + p = _Text(); p.feed(raw); blocks = p.close() + if not blocks: + continue + head = next((t for k, t in blocks if k == "h"), "Part %d" % (idx + 1)) + part, size, n = [], 0, 0 + for k, t in blocks + [("end", "")]: + if k != "end": + line = ("

%s

" if k == "h" else "

%s

") % html.escape(t) + part.append(line); size += len(line.encode("utf-8")) + if (size >= PART_MAX or k == "end") and part: + n += 1 + pages.append(("c%03d-%d.htm" % (idx, n), head if n == 1 else "%s (%d)" % (head, n), part)) + part, size = [], 0 + for i, (fn, ptitle, part) in enumerate(pages): + nav = ['Contents'] + if i > 0: + nav.insert(0, 'Previous' % pages[i - 1][0]) + if i + 1 < len(pages): + nav.append('Next' % pages[i + 1][0]) + open(os.path.join(d, fn), "w", encoding="utf-8").write(_page(ptitle, "\n".join(part), nav)) + toc = "\n".join('

%s

' % (fn, html.escape(t)) for fn, t, _ in pages) + open(os.path.join(d, "index.htm"), "w", encoding="utf-8").write( + _page(title, toc, ['Library'])) + return title, "/%s/%s/index.htm" % (LIB, slug(title)), len(pages) + + +def pack_text(src, card): + d = os.path.join(card, LIB) + os.makedirs(d, exist_ok=True) + name = os.path.basename(src) + shutil.copyfile(src, os.path.join(d, name)) + return os.path.splitext(name)[0], "/%s/%s" % (LIB, name), 1 + + +def write_home(card): + """List every book (folder with index.htm) and text file under /library.""" + d = os.path.join(card, LIB) + rows = [] + for e in sorted(os.listdir(d)) if os.path.isdir(d) else []: + p = os.path.join(d, e) + if os.path.isdir(p) and os.path.exists(os.path.join(p, "index.htm")): + t = re.search(r"(.*?)", open(os.path.join(p, "index.htm"), encoding="utf-8").read()) + rows.append((html.unescape(t.group(1)) if t else e, "/%s/%s/index.htm" % (LIB, e))) + elif e.lower().endswith((".txt", ".md")): + rows.append((os.path.splitext(e)[0], "/%s/%s" % (LIB, e))) + body = "\n".join('

%s

' % (u, html.escape(t)) for t, u in rows) or "

Nothing here yet.

" + open(os.path.join(card, "home.htm"), "w", encoding="utf-8").write(_page("Library", body, [])) + return len(rows) + + +def main(argv): + if len(argv) < 3: + sys.exit(__doc__) + card, srcs = argv[1], argv[2:] + if not os.path.isdir(card): + sys.exit("not a directory: " + card) + for s in srcs: + low = s.lower() + if low.endswith(".epub"): + t, url, n = pack_epub(s, card) + print("%s: %d pages -> %s" % (t, n, url)) + elif low.endswith((".txt", ".md")): + t, url, _ = pack_text(s, card) + print("%s -> %s" % (t, url)) + else: + print("skipped (not .epub/.txt/.md): " + s) + print("home.htm lists %d item(s)" % write_home(card)) + + +if __name__ == "__main__": + main(sys.argv) diff --git a/src/ui-touch/UITask.cpp b/src/ui-touch/UITask.cpp index 0fadcac..4bd837d 100644 --- a/src/ui-touch/UITask.cpp +++ b/src/ui-touch/UITask.cpp @@ -26513,11 +26513,46 @@ static void fmOpenImage(const char* name) { } // ----- row interaction ----- +#if defined(ESP32) && defined(MULTI_TRANSPORT_COMPANION) +static void openReaderPage(const char* initial_url); // the Reader, further down +// A document on the card opens in the Reader: HTML always (its links work there, so a +// folder of pages is browsable offline), and plain text or markdown too big for the +// editor (it pages through a book of any size). Small text files keep the editor. +static bool fmOpenInReader(const char* name) { + bool on_card = false; +#if CAP_SD || defined(TLORA_PAGER) + on_card = s_fm_fs == &SD; +#elif defined(HAS_TANMATSU) || defined(HAS_TDISPLAY_P4) + on_card = s_fm_fs == &SD_MMC; +#endif + if (!on_card) return false; + const size_t len = strlen(name); + auto ends = [&](const char* ext) { const size_t e = strlen(ext); return len > e && strcasecmp(name + len - e, ext) == 0; }; + const bool html = ends(".htm") || ends(".html"); + const bool text = ends(".txt") || ends(".md"); + if (!html && !text) return false; + char path[200]; + fmFullPath(name, path, sizeof path); + if (text) { + File f = s_fm_fs->open(path, "r"); + const size_t sz = f ? f.size() : 0; + if (f) f.close(); + if (sz <= FM_EDIT_MAX) return false; // small: the editor, as before + } + char url[210]; + snprintf(url, sizeof url, "sd:%s", path); + openReaderPage(url); + return true; +} +#else +static bool fmOpenInReader(const char*) { return false; } +#endif static void fmRowClickCb(lv_event_t* e) { if (lv_event_get_code(e) != LV_EVENT_CLICKED) return; FmRowData* rd = (FmRowData*)lv_obj_get_user_data(lv_event_get_target(e)); if (!rd) return; if (rd->isdir) fmEnterDir(rd->name); + else if (fmOpenInReader(rd->name)) {} // HTML, and big text, read in the Reader #if CAP_SOUND_FILES else if (fmIsAudio(rd->name)) fmOpenAudio(rd->name); // .wav -> notification-sound chooser #endif @@ -27523,8 +27558,18 @@ class ReaderBufferStream final : public Stream { enum class ReaderLocalResult : uint8_t { Ok, NoStorage, NotFound, ReadFailed }; +// A plain-text or markdown file on the card: read as a window of the file instead of a +// whole page, so a book of any size pages through ("Next page" links carry ?o=offset). +static const size_t kReaderTxtWindow = 16 * 1024; +static bool readerIsTextPath(const char* url) { + const char* q = strchr(url, '?'); + const size_t n = q ? (size_t)(q - url) : strlen(url); + auto ends = [&](const char* ext) { const size_t e = strlen(ext); return n >= e && strncasecmp(url + n - e, ext, e) == 0; }; + return ends(".txt") || ends(".md"); +} static ReaderLocalResult readerReadLocal(const char* url, uint8_t* raw, size_t capacity, - size_t* total, uint32_t generation) { + size_t* total, uint32_t generation, + size_t* file_size = nullptr, size_t* window_at = nullptr) { fs::FS* storage = nullptr; bool storage_claimed = false; auto releaseStorage = [&]() { @@ -27550,7 +27595,12 @@ static ReaderLocalResult readerReadLocal(const char* url, uint8_t* raw, size_t c char path[300]; snprintf(path, sizeof path, "%s", url + 3); char* query = strchr(path, '?'); - if (query) *query = 0; + size_t offset = 0; + if (query) { + const char* o = strstr(query + 1, "o="); + if (o) offset = (size_t)strtoul(o + 2, nullptr, 10); + *query = 0; + } if (path[0] != '/') { releaseStorage(); return ReaderLocalResult::NotFound; } markSdIo(); @@ -27561,6 +27611,14 @@ static ReaderLocalResult readerReadLocal(const char* url, uint8_t* raw, size_t c return ReaderLocalResult::NotFound; } + if (window_at) { // a text window: start at the offset, read one window + const size_t fs = file.size(); + if (file_size) *file_size = fs; + if (offset > fs) offset = fs; + if (offset && !file.seek(offset)) { file.close(); releaseStorage(); return ReaderLocalResult::ReadFailed; } + *window_at = offset; + if (capacity > kReaderTxtWindow) capacity = kReaderTxtWindow; + } bool read_failed = false; while (*total < capacity && file.available()) { const size_t request = ((capacity - *total) < 4096) ? (capacity - *total) : 4096; @@ -27587,6 +27645,8 @@ static void readerFetchTaskFn(void*) { char result_msg[sizeof s_reader_msg] = ""; const bool local = readerCiPrefix(url, strlen(url), "sd:/"); const bool https = readerCiPrefix(url, strlen(url), "https://"); + const bool text_mode = local && readerIsTextPath(url); + size_t txt_size = 0, txt_at = 0; // Big contiguous PSRAM is scarce on the 2 MB V4 (a chat's message ring fragments it), // so fall back to smaller buffers — a typical text page still fits in 48 KB. size_t rawcap = kReaderRawCap; @@ -27598,7 +27658,8 @@ static void readerFetchTaskFn(void*) { ReaderLocalResult local_result = ReaderLocalResult::NoStorage; if (raw) { if (local) { - local_result = readerReadLocal(url, raw, rawcap - 1, &total, generation); + local_result = readerReadLocal(url, raw, rawcap - 1, &total, generation, + text_mode ? &txt_size : nullptr, text_mode ? &txt_at : nullptr); source_ok = local_result == ReaderLocalResult::Ok; } else { HTTPClient httpc; @@ -27623,7 +27684,43 @@ static void readerFetchTaskFn(void*) { else if (source_ok) { if (!s_reader_text) s_reader_text = (char*)heap_caps_malloc(kReaderTextCap, MALLOC_CAP_SPIRAM); if (!s_reader_links) s_reader_links = (ReaderLink*)heap_caps_malloc(sizeof(ReaderLink) * kReaderMaxLinks, MALLOC_CAP_SPIRAM); - if (s_reader_text) { size_t parsed_links = 0; + if (s_reader_text && text_mode) { + // One window of the text, cut back to its last line (or a whole UTF-8 character) unless + // it is the end of the file, then links to the previous and next windows. + size_t n = total; + const bool eof = txt_at + total >= txt_size; + if (!eof) { + size_t cut = n; + while (cut > n / 2 && raw[cut - 1] != '\n') --cut; + if (cut > n / 2) n = cut; + else { while (n && ((uint8_t)raw[n - 1] & 0xC0) == 0x80) --n; if (n && (uint8_t)raw[n - 1] >= 0xC0) --n; } + } + char base[300]; + snprintf(base, sizeof base, "%s", url); + if (char* q = strchr(base, '?')) *q = 0; + size_t o = (size_t)snprintf(s_reader_text, kReaderTextCap, "%u%%\n\n", + txt_size ? (unsigned)((txt_at * 100) / txt_size) : 0u); + const size_t room = kReaderTextCap - o - 64; + if (n > room) n = room; + memcpy(s_reader_text + o, raw, n); + o += n; + s_reader_text[o] = 0; + int nl = 0; + auto addLink = [&](const char* label, size_t at) { + if (!s_reader_links || nl >= kReaderMaxLinks) return; + o += (size_t)snprintf(s_reader_text + o, kReaderTextCap - o, "\n\n"); + ReaderLink& L = s_reader_links[nl++]; + L.start = (uint32_t)o; + o += (size_t)snprintf(s_reader_text + o, kReaderTextCap - o, "%s", label); + L.end = (uint32_t)o; + snprintf(L.href, sizeof L.href, "%s?o=%u", base, (unsigned)at); + }; + if (!eof) addLink(TR("Next page"), txt_at + n); + if (txt_at) addLink(TR("Previous page"), txt_at > kReaderTxtWindow ? txt_at - kReaderTxtWindow : 0); + if (txt_at) addLink(TR("Back to the start"), 0); + s_reader_nlinks = nl; + snprintf(result_msg, sizeof result_msg, "%s", url); result_ok = true; + } else if (s_reader_text) { size_t parsed_links = 0; size_t tl = ReaderContent::htmlToText((const char*)raw, total, s_reader_text, kReaderTextCap, url, s_reader_links, kReaderMaxLinks, &parsed_links); s_reader_nlinks = (int)parsed_links;