shade/wadamesh
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

reader: page through big text files, open card documents from Files, and a tool to fill a card

Browse Source 
Asked for on Discord (prepper use: books and documentation offline on the card).

- The Reader opens a .txt or .md file on the card as a 16 KB window at an
  offset carried in the URL (sd:/x.txt?o=49152), cut back to a line (or a whole
  UTF-8 character), with Next page / Previous page / Back to the start links and
  the position in percent. A book of any size pages through; Back works as usual.
- Files opens HTML on the card in the Reader (its links work there, so a folder of
  pages browses offline) and text or markdown too big for the 8 KB editor; small
  text files keep the editor.
- scripts/sdpack.py (standard library only): an EPUB becomes /library/<name>/ with
  one page per chapter (long chapters split under 30 KB, linked in order) and a
  contents page; .txt/.md are copied as they are; /home.htm lists everything, so
  the Reader, which opens sd:/home.htm, starts on the library. Tested with a
  generated EPUB.
ZIM (Wikipedia) files are not read on the device: their clusters are zstd/xz
blobs of 1-2 MB and the files run to tens of GB on FAT32; converting on a
computer keeps the device side small.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kaj SchittecatandClaude Opus 5.5 committed 2026-10-13 11:48:26 +02:00
1 parent d0b1f58024
commit bec56ba814
2 files changed
+275 -4
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

scripts/sdpack.py
Executable
+174
View File
@@ -0,0 +1,174 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""Put books and documents on a wadamesh SD card for the on-device Reader.
The Reader opens sd:/home.htm when it starts, follows links between pages on the
card without Wi-Fi, and pages through large .txt / .md files on its own. This
tool fills a card for it:
python3 scripts/sdpack.py /Volumes/SDCARD book.epub notes.txt manual.md
- An EPUB becomes /library/<name>/ : one small page per chapter (split further when
a chapter is long, each part linked to the next) plus index.htm, its contents.
- A .txt or .md file is copied to /library/ as it is: the Reader pages through it.
- /home.htm is (re)written to list everything in /library, so the Reader opens on it.
Only the Python standard library. The card must be FAT32 (the firmware does not
read exFAT). Pages stay well under the Reader's limits (it reads up to 48 KB of a
page on the smallest boards and shows up to 280 links).
"""
import html, os, posixpath, re, shutil, sys, zipfile
from html.parser import HTMLParser
PART_MAX = 30 * 1024 # bytes of text per page: under the Reader's smallest buffer
LIB = "library"
def slug(name):
s = re.sub(r"[^A-Za-z0-9]+", "-", name).strip("-").lower()
return (s or "book")[:40]
class _Text(HTMLParser):
"""XHTML chapter -> blocks of (kind, text): headings and paragraphs, nothing else."""
BLOCK = {"p", "div", "li", "br", "tr", "blockquote", "pre"}
HEAD = {"h1", "h2", "h3", "h4", "h5", "h6"}
SKIP = {"script", "style", "head", "title"}
def __init__(self):
super().__init__(convert_charrefs=True)
self.blocks, self.cur, self.kind, self.skip = [], [], "p", 0
def _flush(self):
t = re.sub(r"\s+", " ", "".join(self.cur)).strip()
if t:
self.blocks.append((self.kind, t))
self.cur, self.kind = [], "p"
def handle_starttag(self, tag, attrs):
if tag in self.SKIP:
self.skip += 1
elif tag in self.HEAD:
self._flush(); self.kind = "h"
elif tag in self.BLOCK:
self._flush()
def handle_endtag(self, tag):
if tag in self.SKIP:
self.skip = max(0, self.skip - 1)
elif tag in self.HEAD or tag in self.BLOCK:
self._flush()
def handle_data(self, data):
if not self.skip:
self.cur.append(data)
def close(self):
super().close(); self._flush()
return self.blocks
def _page(title, body, nav):
return ("<!doctype html><html><head><meta charset=\"utf-8\"><title>%s</title></head><body>\n"
"<h1>%s</h1>\n%s\n<p>%s</p>\n</body></html>\n") % (
html.escape(title), html.escape(title), body, " | ".join(nav))
def _opf(z):
container = z.read("META-INF/container.xml").decode("utf-8", "replace")
opf_path = re.search(r'full-path="([^"]+)"', container).group(1)
opf = z.read(opf_path).decode("utf-8", "replace")
base = posixpath.dirname(opf_path)
title = re.search(r"<dc:title[^>]*>(.*?)</dc:title>", opf, re.S)
items = {m.group(1): m.group(2) for m in
re.finditer(r'<item\b[^>]*?id="([^"]+)"[^>]*?href="([^"]+)"', opf)}
items.update({m.group(2): m.group(1) for m in
re.finditer(r'<item\b[^>]*?href="([^"]+)"[^>]*?id="([^"]+)"', opf)})
spine = re.findall(r'<itemref\b[^>]*?idref="([^"]+)"', opf)
files = [posixpath.normpath(posixpath.join(base, items[i])) for i in spine if i in items]
return (html.unescape(title.group(1).strip()) if title else None), files
def pack_epub(src, card):
z = zipfile.ZipFile(src)
title, files = _opf(z)
title = title or os.path.splitext(os.path.basename(src))[0]
d = os.path.join(card, LIB, slug(title))
os.makedirs(d, exist_ok=True)
pages = [] # (file name, page title)
for idx, f in enumerate(files):
try:
raw = z.read(f).decode("utf-8", "replace")
except KeyError:
continue
p = _Text(); p.feed(raw); blocks = p.close()
if not blocks:
continue
head = next((t for k, t in blocks if k == "h"), "Part %d" % (idx + 1))
part, size, n = [], 0, 0
for k, t in blocks + [("end", "")]:
if k != "end":
line = ("<h2>%s</h2>" if k == "h" else "<p>%s</p>") % html.escape(t)
part.append(line); size += len(line.encode("utf-8"))
if (size >= PART_MAX or k == "end") and part:
n += 1
pages.append(("c%03d-%d.htm" % (idx, n), head if n == 1 else "%s (%d)" % (head, n), part))
part, size = [], 0
for i, (fn, ptitle, part) in enumerate(pages):
nav = ['<a href="index.htm">Contents</a>']
if i > 0:
nav.insert(0, '<a href="%s">Previous</a>' % pages[i - 1][0])
if i + 1 < len(pages):
nav.append('<a href="%s">Next</a>' % pages[i + 1][0])
open(os.path.join(d, fn), "w", encoding="utf-8").write(_page(ptitle, "\n".join(part), nav))
toc = "\n".join('<p><a href="%s">%s</a></p>' % (fn, html.escape(t)) for fn, t, _ in pages)
open(os.path.join(d, "index.htm"), "w", encoding="utf-8").write(
_page(title, toc, ['<a href="/home.htm">Library</a>']))
return title, "/%s/%s/index.htm" % (LIB, slug(title)), len(pages)
def pack_text(src, card):
d = os.path.join(card, LIB)
os.makedirs(d, exist_ok=True)
name = os.path.basename(src)
shutil.copyfile(src, os.path.join(d, name))
return os.path.splitext(name)[0], "/%s/%s" % (LIB, name), 1
def write_home(card):
"""List every book (folder with index.htm) and text file under /library."""
d = os.path.join(card, LIB)
rows = []
for e in sorted(os.listdir(d)) if os.path.isdir(d) else []:
p = os.path.join(d, e)
if os.path.isdir(p) and os.path.exists(os.path.join(p, "index.htm")):
t = re.search(r"<title>(.*?)</title>", open(os.path.join(p, "index.htm"), encoding="utf-8").read())
rows.append((html.unescape(t.group(1)) if t else e, "/%s/%s/index.htm" % (LIB, e)))
elif e.lower().endswith((".txt", ".md")):
rows.append((os.path.splitext(e)[0], "/%s/%s" % (LIB, e)))
body = "\n".join('<p><a href="%s">%s</a></p>' % (u, html.escape(t)) for t, u in rows) or "<p>Nothing here yet.</p>"
open(os.path.join(card, "home.htm"), "w", encoding="utf-8").write(_page("Library", body, []))
return len(rows)
def main(argv):
if len(argv) < 3:
sys.exit(__doc__)
card, srcs = argv[1], argv[2:]
if not os.path.isdir(card):
sys.exit("not a directory: " + card)
for s in srcs:
low = s.lower()
if low.endswith(".epub"):
t, url, n = pack_epub(s, card)
print("%s: %d pages -> %s" % (t, n, url))
elif low.endswith((".txt", ".md")):
t, url, _ = pack_text(s, card)
print("%s -> %s" % (t, url))
else:
print("skipped (not .epub/.txt/.md): " + s)
print("home.htm lists %d item(s)" % write_home(card))
if __name__ == "__main__":
main(sys.argv)
src/ui-touch/UITask.cpp
+101 -4
View File
@@ -26513,11 +26513,46 @@ static void fmOpenImage(const char* name) {
}
// ----- row interaction -----
#if defined(ESP32) && defined(MULTI_TRANSPORT_COMPANION)
static void openReaderPage(const char* initial_url); // the Reader, further down
// A document on the card opens in the Reader: HTML always (its links work there, so a
// folder of pages is browsable offline), and plain text or markdown too big for the
// editor (it pages through a book of any size). Small text files keep the editor.
static bool fmOpenInReader(const char* name) {
bool on_card = false;
#if CAP_SD || defined(TLORA_PAGER)
on_card = s_fm_fs == &SD;
#elif defined(HAS_TANMATSU) || defined(HAS_TDISPLAY_P4)
on_card = s_fm_fs == &SD_MMC;
#endif
if (!on_card) return false;
const size_t len = strlen(name);
auto ends = [&](const char* ext) { const size_t e = strlen(ext); return len > e && strcasecmp(name + len - e, ext) == 0; };
const bool html = ends(".htm") || ends(".html");
const bool text = ends(".txt") || ends(".md");
if (!html && !text) return false;
char path[200];
fmFullPath(name, path, sizeof path);
if (text) {
File f = s_fm_fs->open(path, "r");
const size_t sz = f ? f.size() : 0;
if (f) f.close();
if (sz <= FM_EDIT_MAX) return false; // small: the editor, as before
}
char url[210];
snprintf(url, sizeof url, "sd:%s", path);
openReaderPage(url);
return true;
}
#else
static bool fmOpenInReader(const char*) { return false; }
#endif
static void fmRowClickCb(lv_event_t* e) {
if (lv_event_get_code(e) != LV_EVENT_CLICKED) return;
FmRowData* rd = (FmRowData*)lv_obj_get_user_data(lv_event_get_target(e));
if (!rd) return;
if (rd->isdir) fmEnterDir(rd->name);
else if (fmOpenInReader(rd->name)) {} // HTML, and big text, read in the Reader
#if CAP_SOUND_FILES
else if (fmIsAudio(rd->name)) fmOpenAudio(rd->name); // .wav -> notification-sound chooser
#endif
@@ -27523,8 +27558,18 @@ class ReaderBufferStream final : public Stream {
enum class ReaderLocalResult : uint8_t { Ok, NoStorage, NotFound, ReadFailed };
// A plain-text or markdown file on the card: read as a window of the file instead of a
// whole page, so a book of any size pages through ("Next page" links carry ?o=offset).
static const size_t kReaderTxtWindow = 16 * 1024;
static bool readerIsTextPath(const char* url) {
const char* q = strchr(url, '?');
const size_t n = q ? (size_t)(q - url) : strlen(url);
auto ends = [&](const char* ext) { const size_t e = strlen(ext); return n >= e && strncasecmp(url + n - e, ext, e) == 0; };
return ends(".txt") || ends(".md");
}
static ReaderLocalResult readerReadLocal(const char* url, uint8_t* raw, size_t capacity,
size_t* total, uint32_t generation) {
size_t* total, uint32_t generation,
size_t* file_size = nullptr, size_t* window_at = nullptr) {
fs::FS* storage = nullptr;
bool storage_claimed = false;
auto releaseStorage = [&]() {
@@ -27550,7 +27595,12 @@ static ReaderLocalResult readerReadLocal(const char* url, uint8_t* raw, size_t c
char path[300];
snprintf(path, sizeof path, "%s", url + 3);
char* query = strchr(path, '?');
if (query) *query = 0;
size_t offset = 0;
if (query) {
const char* o = strstr(query + 1, "o=");
if (o) offset = (size_t)strtoul(o + 2, nullptr, 10);
*query = 0;
}
if (path[0] != '/') { releaseStorage(); return ReaderLocalResult::NotFound; }
markSdIo();
@@ -27561,6 +27611,14 @@ static ReaderLocalResult readerReadLocal(const char* url, uint8_t* raw, size_t c
return ReaderLocalResult::NotFound;
}
if (window_at) { // a text window: start at the offset, read one window
const size_t fs = file.size();
if (file_size) *file_size = fs;
if (offset > fs) offset = fs;
if (offset && !file.seek(offset)) { file.close(); releaseStorage(); return ReaderLocalResult::ReadFailed; }
*window_at = offset;
if (capacity > kReaderTxtWindow) capacity = kReaderTxtWindow;
}
bool read_failed = false;
while (*total < capacity && file.available()) {
const size_t request = ((capacity - *total) < 4096) ? (capacity - *total) : 4096;
@@ -27587,6 +27645,8 @@ static void readerFetchTaskFn(void*) {
char result_msg[sizeof s_reader_msg] = "";
const bool local = readerCiPrefix(url, strlen(url), "sd:/");
const bool https = readerCiPrefix(url, strlen(url), "https://");
const bool text_mode = local && readerIsTextPath(url);
size_t txt_size = 0, txt_at = 0;
// Big contiguous PSRAM is scarce on the 2 MB V4 (a chat's message ring fragments it),
// so fall back to smaller buffers — a typical text page still fits in 48 KB.
size_t rawcap = kReaderRawCap;
@@ -27598,7 +27658,8 @@ static void readerFetchTaskFn(void*) {
ReaderLocalResult local_result = ReaderLocalResult::NoStorage;
if (raw) {
if (local) {
local_result = readerReadLocal(url, raw, rawcap - 1, &total, generation);
local_result = readerReadLocal(url, raw, rawcap - 1, &total, generation,
text_mode ? &txt_size : nullptr, text_mode ? &txt_at : nullptr);
source_ok = local_result == ReaderLocalResult::Ok;
} else {
HTTPClient httpc;
@@ -27623,7 +27684,43 @@ static void readerFetchTaskFn(void*) {
else if (source_ok) {
if (!s_reader_text) s_reader_text = (char*)heap_caps_malloc(kReaderTextCap, MALLOC_CAP_SPIRAM);
if (!s_reader_links) s_reader_links = (ReaderLink*)heap_caps_malloc(sizeof(ReaderLink) * kReaderMaxLinks, MALLOC_CAP_SPIRAM);
if (s_reader_text) { size_t parsed_links = 0;
if (s_reader_text && text_mode) {
// One window of the text, cut back to its last line (or a whole UTF-8 character) unless
// it is the end of the file, then links to the previous and next windows.
size_t n = total;
const bool eof = txt_at + total >= txt_size;
if (!eof) {
size_t cut = n;
while (cut > n / 2 && raw[cut - 1] != '\n') --cut;
if (cut > n / 2) n = cut;
else { while (n && ((uint8_t)raw[n - 1] & 0xC0) == 0x80) --n; if (n && (uint8_t)raw[n - 1] >= 0xC0) --n; }
}
char base[300];
snprintf(base, sizeof base, "%s", url);
if (char* q = strchr(base, '?')) *q = 0;
size_t o = (size_t)snprintf(s_reader_text, kReaderTextCap, "%u%%\n\n",
txt_size ? (unsigned)((txt_at * 100) / txt_size) : 0u);
const size_t room = kReaderTextCap - o - 64;
if (n > room) n = room;
memcpy(s_reader_text + o, raw, n);
o += n;
s_reader_text[o] = 0;
int nl = 0;
auto addLink = [&](const char* label, size_t at) {
if (!s_reader_links || nl >= kReaderMaxLinks) return;
o += (size_t)snprintf(s_reader_text + o, kReaderTextCap - o, "\n\n");
ReaderLink& L = s_reader_links[nl++];
L.start = (uint32_t)o;
o += (size_t)snprintf(s_reader_text + o, kReaderTextCap - o, "%s", label);
L.end = (uint32_t)o;
snprintf(L.href, sizeof L.href, "%s?o=%u", base, (unsigned)at);
};
if (!eof) addLink(TR("Next page"), txt_at + n);
if (txt_at) addLink(TR("Previous page"), txt_at > kReaderTxtWindow ? txt_at - kReaderTxtWindow : 0);
if (txt_at) addLink(TR("Back to the start"), 0);
s_reader_nlinks = nl;
snprintf(result_msg, sizeof result_msg, "%s", url); result_ok = true;
} else if (s_reader_text) { size_t parsed_links = 0;
size_t tl = ReaderContent::htmlToText((const char*)raw, total, s_reader_text, kReaderTextCap,
url, s_reader_links, kReaderMaxLinks, &parsed_links);
s_reader_nlinks = (int)parsed_links;