Pack Reader words to retain Full firmware flash headroom

src/helpers/bible/ReaderLookup.h

@@ -30,6 +30,23 @@ template <typename Offset> struct CorpusT {
   uint16_t block_count;
   uint16_t verse_count;
 };
+// Keep the original plain corpus aggregate and its seven-field initializer.
+// Token metadata is static flash data; no decoded block or heap is retained.
+template <typename Offset> struct TokenCorpusT : CorpusT<Offset> {
+  const uint8_t* words;
+  uint16_t words_size;
+  const uint16_t* word_offsets;
+  uint16_t word_count;
+
+  constexpr TokenCorpusT(const char* translation, const char* attribution,
+      const uint8_t* data, uint32_t data_size, const BlockIndexT<Offset>* blocks,
+      uint16_t block_count, uint16_t verse_count, const uint8_t* words,
+      uint16_t words_size, const uint16_t* word_offsets, uint16_t word_count)
+      : CorpusT<Offset>{translation, attribution, data, data_size, blocks,
+                        block_count, verse_count},
+        words(words), words_size(words_size), word_offsets(word_offsets),
+        word_count(word_count) {}
+};
 // Keep the original general-purpose types available. Generated corpora select
 // a narrower offset only when every block start fits, without a runtime branch.
 using BlockIndex = BlockIndexT<uint32_t>;
@@ -90,8 +107,10 @@ inline ParseResult parse(const char* command, Reference& ref) {
 // stay in memory-mapped flash on both target families; there is no heap/cache.
 template <typename Offset>
 inline LookupResult lookup(const CorpusT<Offset>& corpus, Reference ref,
-                           char* scratch, size_t capacity, const char*& text) {
+                           char* scratch, size_t capacity, const char*& text,
+                           size_t* decoded_size = nullptr) {
   text = nullptr;
+  if (decoded_size) *decoded_size = 0;
   uint16_t index;
   if (!verseNumber(ref, index)) return LookupResult::Invalid;
   if (!scratch || capacity < kBlockSize || !corpus.data || !corpus.blocks ||
@@ -111,6 +130,7 @@ inline LookupResult lookup(const CorpusT<Offset>& corpus, Reference ref,
                             block.packed) != TINF_OK || produced != block.plain) {
     return LookupResult::Corrupt;
   }
+  if (decoded_size) *decoded_size = produced;
   const char* current = scratch;
   const char* end = scratch + produced;
   for (unsigned v = block.first_verse; v <= index; ++v) {
@@ -126,4 +146,57 @@ inline LookupResult lookup(const CorpusT<Offset>& corpus, Reference ref,
   return LookupResult::Found;
 }
 
+// Expand only the selected verse after its token block in the same 4 KiB
+// scratch. The packer reserves this headroom for every tokenized verse.
+template <typename Offset>
+inline LookupResult lookup(const TokenCorpusT<Offset>& corpus, Reference ref,
+                           char* scratch, size_t capacity, const char*& text) {
+  size_t produced;
+  const LookupResult result = lookup(static_cast<const CorpusT<Offset>&>(corpus),
+                                    ref, scratch, capacity, text, &produced);
+  if (result != LookupResult::Found) return result;
+  const uint8_t* encoded = reinterpret_cast<const uint8_t*>(text);
+  text = nullptr;
+  if (!corpus.words || !corpus.word_offsets || corpus.word_count == 0
+      || corpus.word_count > 128 || corpus.words_size == 0
+      || corpus.word_offsets[0] != 0
+      || corpus.word_offsets[corpus.word_count] != corpus.words_size)
+    return LookupResult::Corrupt;
+  bool tokenized = false;
+  for (const uint8_t* p = encoded; *p; ++p) {
+    if (*p < 32 || *p == 127) return LookupResult::Corrupt;
+    if (*p >= 128) tokenized = true;
+  }
+  // Literal fallback retains the original maximum-length verse allowance.
+  if (!tokenized) {
+    text = reinterpret_cast<const char*>(encoded);
+    return LookupResult::Found;
+  }
+  char* output = scratch + produced;
+  size_t available = kBlockSize - produced;
+  char* dest = output;
+  for (; *encoded; ++encoded) {
+    const uint8_t* word = encoded;
+    size_t length = 1;
+    if (*encoded >= 128) {
+      const unsigned index = *encoded - 128;
+      if (index >= corpus.word_count) return LookupResult::Corrupt;
+      const uint16_t start = corpus.word_offsets[index];
+      const uint16_t end = corpus.word_offsets[index + 1];
+      if (start >= end || end > corpus.words_size) return LookupResult::Corrupt;
+      word = corpus.words + start;
+      length = end - start;
+    }
+    if (length >= available) return LookupResult::Corrupt; // Retain the NUL.
+    for (size_t i = 0; i < length; ++i) {
+      if (word[i] < 32 || word[i] >= 127) return LookupResult::Corrupt;
+      *dest++ = static_cast<char>(word[i]);
+    }
+    available -= length;
+  }
+  *dest = 0;
+  text = output;
+  return LookupResult::Found;
+}
+
 }} // namespace mesh::bible

test/fixtures/companion_reader/test_reader_lookup.cpp

@@ -24,6 +24,57 @@ static void checkOffsetWidth(uint32_t offset) {
   assert(text == nullptr);
 }
 
+static void checkTokenBounds() {
+  struct Guarded { uint32_t pre; char scratch[kBlockSize]; uint32_t post; } buf;
+  const uint8_t words[] = {'a', 'b', 'c'};
+  const uint16_t offsets[] = {0, 3};
+  auto check = [&](const std::vector<uint8_t>& encoded, const uint8_t* dictionary,
+                   uint16_t size, const uint16_t* starts, uint16_t count,
+                   LookupResult expected, const char* expected_text = nullptr) {
+    assert(encoded.size() <= kBlockSize);
+    const uint16_t length = encoded.size();
+    std::vector<uint8_t> data = {1, uint8_t(length), uint8_t(length >> 8),
+        uint8_t(~length), uint8_t(~length >> 8)};
+    data.insert(data.end(), encoded.begin(), encoded.end());
+    const BlockIndexT<uint16_t> block = {0, uint16_t(data.size()), length, 0};
+    const TokenCorpusT<uint16_t> corpus = {"test", "test", data.data(),
+        uint32_t(data.size()), &block, 1, kVerseCount, dictionary, size, starts, count};
+    buf.pre = 0x11223344;buf.post = 0xaabbccdd;
+    const char* text = reinterpret_cast<const char*>(uintptr_t(1));
+    assert(lookup(corpus, {1, 1}, buf.scratch, sizeof(buf.scratch), text) == expected);
+    assert(buf.pre == 0x11223344 && buf.post == 0xaabbccdd);
+    if (expected_text) assert(text && strcmp(text, expected_text) == 0);
+    else assert(text == nullptr);
+  };
+  check({128, 0}, words, 3, offsets, 1, LookupResult::Found, "abc");
+  check({255, 0}, words, 3, offsets, 1, LookupResult::Corrupt);
+  check({128, 0}, nullptr, 3, offsets, 1, LookupResult::Corrupt);
+  check({128, 0}, words, 0, offsets, 1, LookupResult::Corrupt);
+  check({128, 0}, words, 3, nullptr, 1, LookupResult::Corrupt);
+  check({128, 0}, words, 3, offsets, 0, LookupResult::Corrupt);
+  check({128, 0}, words, 3, offsets, 129, LookupResult::Corrupt);
+  for (const auto& bad : {std::vector<uint16_t>{1, 3}, {0, 4}, {0, 0},
+                          {0, 4, 3}, {0, 0, 3}})
+    check({128, 0}, words, 3, bad.data(), uint16_t(bad.size() - 1), LookupResult::Corrupt);
+  const uint16_t decreasing[] = {0, 3, 2, 3};
+  check({129, 0}, words, 3, decreasing, 3, LookupResult::Corrupt);
+  for (uint8_t invalid : {uint8_t(0), uint8_t(31), uint8_t(127), uint8_t(128)}) {
+    const uint8_t invalid_words[] = {'a', invalid, 'c'};
+    check({128, 0}, invalid_words, 3, offsets, 1, LookupResult::Corrupt);
+  }
+  check({'x', 31, 0}, words, 3, offsets, 1, LookupResult::Corrupt);
+  check({'x', 127, 0}, words, 3, offsets, 1, LookupResult::Corrupt);
+  std::vector<uint8_t> exact(kBlockSize - 4, 0);exact[0] = 128;
+  check(exact, words, 3, offsets, 1, LookupResult::Found, "abc");
+  exact.push_back(0);
+  check(exact, words, 3, offsets, 1, LookupResult::Corrupt);
+  std::vector<uint8_t> literal(kBlockSize, 'q');literal.back() = 0;
+  check(literal, words, 3, offsets, 1, LookupResult::Found,
+        std::string(kBlockSize - 1, 'q').c_str());
+  literal[0] = 128;
+  check(literal, words, 3, offsets, 1, LookupResult::Corrupt);
+}
+
 static void drain(Stream& output) {
   unsigned calls = 0;
   while (mesh::readerCommandActive()) {
@@ -90,6 +141,7 @@ int main(int argc, char**) {
   static_assert(sizeof(BlockIndexT<uint32_t>) == 12, "wide index alignment");
   checkOffsetWidth<uint16_t>(UINT16_MAX);
   checkOffsetWidth<uint32_t>(uint32_t(UINT16_MAX) + 1);
+  checkTokenBounds();
   Reference ref = {};
   for (const char* cmd : {"get reader 3:16", " GET\treader 3:16 \t", "get reader 03:16"}) {
     assert(parse(cmd, ref) == ParseResult::Valid && ref.chapter == 3 && ref.verse == 16);

test/test_companion_reader.py

@@ -3,6 +3,7 @@
 import copy
 import importlib.util
 import json
+import os
 from pathlib import Path
 import shutil
 import subprocess
@@ -37,6 +38,7 @@ class CompanionReaderTest(unittest.TestCase):
     def test_block_sizes_roundtrip_all_verses(self):
         for size in (1024, 2048, 4096):
             packed, blocks = packer.pack(self.document, size)
+            words = packer.token_dictionary(packer.source_verses(self.document, size))
             actual = []
             for offset, length, plain_length, first in blocks:
                 self.assertEqual(first, len(actual))
@@ -44,7 +46,12 @@ class CompanionReaderTest(unittest.TestCase):
                 self.assertEqual(len(plain), plain_length)
                 self.assertLessEqual(len(plain), size)
                 self.assertEqual(plain[-1], 0)
-                actual.extend(part.decode("ascii") for part in plain[:-1].split(b"\0"))
+                for encoded in plain[:-1].split(b"\0"):
+                    decoded = b"".join(words[value - 128] if value >= 128 else bytes([value])
+                                       for value in encoded)
+                    if any(value >= 128 for value in encoded):
+                        self.assertLessEqual(len(plain) + len(decoded) + 1, size)
+                    actual.append(decoded.decode("ascii"))
             self.assertEqual(actual, list(self.expected_verses.values()))
 
     def test_arm_reader_decode_frames_fit_existing_loop_stack(self):
@@ -145,11 +152,37 @@ class CompanionReaderTest(unittest.TestCase):
 
     def test_ascii_data_size(self):
         packed, blocks = packer.pack(self.document)
-        self.assertEqual(sum(block[2] for block in blocks), 96568)
-        self.assertEqual(len(blocks), 24)
-        self.assertEqual(len(packed), 38293)
+        self.assertEqual(sum(len(v) + 1 for v in packer.source_verses(self.document)), 96568)
+        self.assertEqual(sum(block[2] for block in blocks), 53806)
+        self.assertEqual(len(blocks), 14)
+        self.assertEqual(len(packed), 32324)
         self.assertEqual(packer.block_index_layout(blocks), ("uint16_t", 8))
-        self.assertEqual(len(packed) + 8 * len(blocks), 38485)
+        words = packer.token_dictionary(packer.source_verses(self.document))
+        self.assertEqual(len(words), 128)
+        self.assertEqual(packer.dictionary_storage(words), 963)
+        self.assertEqual(len(packed) + 8 * len(blocks) + packer.dictionary_storage(words), 33399)
+
+    def test_literal_fallback_and_empty_dictionary_preserve_original_text_limit(self):
+        document = copy.deepcopy(self.document)
+        document["verses"] = dict.fromkeys(packer.references(), None)
+        document["verses"]["1:1"] = "q" * (packer.BLOCK_SIZE - 1)
+        blocks, starts, words = packer.token_blocks(document)
+        self.assertEqual(words, [])
+        self.assertEqual(blocks[0], b"q" * (packer.BLOCK_SIZE - 1) + b"\0")
+        self.assertEqual(starts[:2], [0, 1])
+        with patch.object(packer, "pack", return_value=(b"x", [(0, 1, 4096, 0)])):
+            header = packer.render(document)
+        self.assertIn("using ReaderCorpus = CorpusT<uint16_t>;", header)
+        self.assertNotIn("readerWords[]", header)
+        # A known dictionary cannot force a very long mostly literal verse
+        # to reserve a second copy. Its exact ASCII fallback stays decodable.
+        document = copy.deepcopy(self.document)
+        long_verse = "z the " + "q" * (packer.BLOCK_SIZE - 7)
+        document["verses"]["1:1"] = long_verse
+        blocks, _, words = packer.token_blocks(document)
+        self.assertIn(b" the", words)
+        self.assertEqual(blocks[0], long_verse.encode("ascii") + b"\0")
+        self.assertEqual(len(blocks[0]), packer.BLOCK_SIZE)
 
     def test_index_offsets_widen_before_any_block_start_can_truncate(self):
         self.assertEqual(packer.block_index_layout([(65535, 7, 2, 0)]), ("uint16_t", 8))
@@ -161,7 +194,7 @@ class CompanionReaderTest(unittest.TestCase):
         with patch.object(packer, "pack", return_value=(b"", [(65536, 7, 2, 0)])):
             header = packer.render(self.document)
         self.assertIn("using ReaderBlockIndex = BlockIndexT<uint32_t>;", header)
-        self.assertIn("using ReaderCorpus = CorpusT<uint32_t>;", header)
+        self.assertIn("using ReaderCorpus = TokenCorpusT<uint32_t>;", header)
         self.assertIn("sizeof(ReaderBlockIndex) == 12", header)
         self.assertIn("{65536, 7, 2, 0}", header)
 
@@ -265,12 +298,39 @@ int main() {
         self.assertEqual(source.count("mesh::handleReaderCommand("), 2)
         self.assertLess(terminal.index("mesh::handleReaderCommand("),
                         terminal.index("char local_reply[160]"))
-        self.assertIn("get reader <chapter>[:<verse>[-<last>]] [page] (World English Bible, offline)", terminal)
-        self.assertEqual(terminal.count("#if COMPANION_FEATURE_READER"), 2)
+        help_text = (ROOT / "examples/companion_radio/CompanionHelp.inc").read_text()
+        self.assertIn("get reader <chapter>[:<verse>[-<last>]] [page] (World English Bible, offline)", help_text)
+        self.assertIn('#include "CompanionHelp.inc"', terminal)
+        self.assertIn("mesh::companion::writePackedCompanionHelp(output)", terminal)
+        packed_help = (ROOT / "examples/companion_radio/CompanionHelpPacked.cpp").read_text()
+        self.assertIn('#include "CompanionHelp.inc"', packed_help)
+        self.assertEqual(terminal.count("#if COMPANION_FEATURE_READER"), 1)
+        self.assertEqual(help_text.count("#if COMPANION_FEATURE_READER"), 1)
         rescue = extract_braced(source, "void MyMesh::checkCLIRescueCmd()")
         self.assertLess(rescue.index("mesh::handleReaderCommand(cli_command, output)"),
                         rescue.index("handleCommand(cli_command, 0, reply_buf)"))
 
+    def test_token_corruption_boundaries_under_native_sanitizers(self):
+        cc, cxx = shutil.which("gcc"), shutil.which("g++")
+        if not cc or not cxx:
+            self.skipTest("host GCC and G++ required")
+        source = (FIXTURE / "test_reader_lookup.cpp").read_text()
+        checks = extract_braced(source, "static void checkTokenBounds()")
+        harness = ("#include <helpers/bible/ReaderLookup.h>\n#include <cassert>\n"
+                   "#include <string>\n#include <vector>\nusing namespace mesh::bible;\n"
+                   + checks + "\nint main() { checkTokenBounds(); }\n")
+        sanitizers = [] if os.name == "nt" else [
+            "-fsanitize=address,undefined", "-fno-sanitize-recover=all", "-fno-pie", "-no-pie"]
+        with tempfile.TemporaryDirectory(prefix="mesh-reader-token-bounds-") as directory:
+            directory = Path(directory)
+            cpp, obj, binary = directory / "checks.cpp", directory / "tinf.o", directory / "checks"
+            cpp.write_text(harness)
+            self.run_checked([cc, "-std=c11", "-Os", "-DOTA_TRANSPORT_DEFLATE_TEST=1", *sanitizers, "-c",
+                              str(ROOT / "src/helpers/ota/OtaTinf.c"), "-o", str(obj)])
+            self.run_checked([cxx, "-std=c++17", "-Os", "-Wall", "-Wextra", "-Werror",
+                              *sanitizers, "-I" + str(ROOT / "src"), str(cpp), str(obj), "-o", str(binary)])
+            self.run_checked([str(binary)])
+
     def test_production_bounded_reader_dispatch_preserves_tags_and_local_access(self):
         cc, cxx = shutil.which("gcc"), shutil.which("g++")
         if not cc or not cxx:

tools/bible/README.md

@@ -97,10 +97,30 @@ headings; punctuation and words are preserved as UTF-8 in this source file.
 The packer converts curly quotes/apostrophes to straight ones, en/em dashes to
 hyphens, and nonbreaking spaces to ordinary spaces before compression. No
 letters or words are changed. Any other non-ASCII character is rejected for
-review, never silently dropped or transliterated. This conversion happens on
-the computer, with no firmware conversion buffer or extra decoding step.
+review, never silently dropped or transliterated. This typography conversion
+happens on the computer; the firmware never changes wording or punctuation.
 No NLT or ERV text is included.
 
+The generated copy uses 128 exact, case-sensitive word tokens before DEFLATE.
+Each token retains its original letters and optional preceding space in a
+flash dictionary. Lookup inflates one token block, then expands only the
+selected verse after that block in the same 4 KiB scratch. The packer checks
+that every block plus its largest expanded selection fits; a very long literal
+verse can stay in place to retain the original 4095-byte text limit. Legacy
+plain corpora keep their original format and lookup behavior. Dictionary
+indices, word bounds, printable ASCII and output headroom are checked before
+returning a verse. No heap, filesystem cache or additional decode buffer is
+used.
+
+The pinned 879 verses remain 96,568 ASCII bytes including separators. The
+generated 14-block copy uses 32,324 DEFLATE bytes, 112 block-index bytes, 705
+dictionary bytes and 258 dictionary-offset bytes: 33,399 bytes in total,
+5,086 fewer than the preceding 38,485-byte plain-block copy. This is static
+asset accounting; the complete firmware size also includes the lookup code.
+Tests compare all 879 decoded verses and their terminal output against the
+independently normalized pinned source, including pagination, corruption and
+the existing ARM stack limits.
+
 The text is [public domain](https://ebible.org/engwebp/copyright.htm).
 World English Bible is a trademark of eBible.org; the source must not be
 rewritten while continuing to label it WEB. The source and generated header

tools/bible/pack_reader.py

@@ -1,9 +1,12 @@
 #!/usr/bin/env python3
 """Pack the pinned public-domain source for the offline Reader into DEFLATE blocks."""
 import argparse
+from collections import Counter
+from functools import lru_cache
 import hashlib
 import json
 from pathlib import Path
+import re
 import sys
 import unicodedata
 
@@ -17,6 +20,8 @@ SOURCE = Path(__file__).with_name("reader-web.json")
 HEADER = ROOT / "src/helpers/bible/ReaderData.generated.h"
 BLOCK_SIZE = 4096
 COMPRESSION_ITERATIONS = MAXIMUM_ITERATIONS
+TOKEN_WORDS = 128
+WORD = re.compile(rb" ?[A-Za-z]+")
 ASCII_TYPOGRAPHY = str.maketrans({
     "\u00a0": " ", "\u2013": "-", "\u2014": "-",
     "\u2018": "'", "\u2019": "'", "\u201c": '"', "\u201d": '"',
@@ -54,32 +59,85 @@ def ascii_verse(value, limit):
     return encoded
 
 
-def pack(document, block_size=BLOCK_SIZE):
-    if block_size not in (1024, 2048, 4096):
-        raise ValueError("block size must be 1024, 2048, or 4096")
+def source_verses(document, block_size=BLOCK_SIZE):
     checked_text(document["translation"], 24)
     checked_text(document["attribution"], 256)
     verses = document["verses"]
     if not isinstance(verses, dict) or set(verses) != set(references()):
         raise ValueError("source must explicitly include all 879 Reader references")
+    return [ascii_verse(verses[ref], block_size - 1) if verses[ref] is not None else b""
+            for ref in references()]
+
+
+def token_dictionary(verses):
+    """Choose exact, case-sensitive ASCII words deterministically, including spaces."""
+    counts = Counter(match.group() for verse in verses for match in WORD.finditer(verse))
+    gain = lambda word: (len(word) - 1) * counts[word] - len(word) - 2
+    ordered = sorted(counts, key=lambda word: (gain(word), word), reverse=True)
+    words, size = [], 0
+    for word in ordered:
+        if gain(word) <= 0:
+            break
+        if size + len(word) > 0xffff:
+            continue
+        words.append(word)
+        size += len(word)
+        if len(words) == TOKEN_WORDS:
+            break
+    return words
+
+
+def dictionary_storage(words):
+    return sum(map(len, words)) + 2 * (len(words) + 1) if words else 0
+
+
+def token_blocks(document, block_size=BLOCK_SIZE):
+    if block_size not in (1024, 2048, 4096):
+        raise ValueError("block size must be 1024, 2048, or 4096")
+    verses = source_verses(document, block_size)
+    words = token_dictionary(verses)
+    tokens = {word: bytes([128 + index]) for index, word in enumerate(words)}
     blocks, starts, current = [], [0], bytearray()
-    for index, reference in enumerate(references()):
-        verse = verses[reference]
-        encoded = (ascii_verse(verse, block_size - 1) if verse is not None else b"") + b"\0"
-        if len(current) + len(encoded) > block_size:
+    expansion = 0
+    for index, verse in enumerate(verses):
+        encoded = WORD.sub(lambda match: tokens.get(match.group(), match.group()), verse)
+        required = len(verse) + 1 if encoded != verse else 0
+        # A very long, mostly unique verse may not fit beside its encoded
+        # input. Retain it literally: lookup returns an all-ASCII verse in
+        # place and needs no expansion area, preserving the original limit.
+        if len(encoded) + 1 + required > block_size:
+            encoded, required = verse, 0
+        maximum = max(expansion, required)
+        if len(current) + len(encoded) + 1 + maximum > block_size:
             blocks.append(bytes(current))
             current.clear()
             starts.append(index)
-        current.extend(encoded)
+            expansion = 0
+        current.extend(encoded + b"\0")
+        expansion = max(expansion, required)
+        if len(current) + expansion > block_size:
+            raise AssertionError("Reader encoded block and selected output exceed scratch")
     if current:
         blocks.append(bytes(current))
     if not blocks:
         raise ValueError("source contains no verse text")
+    return blocks, starts, words
+
+
+@lru_cache(maxsize=128)
+def compress_block(block):
+    return raw_deflate_compress(block, numiterations=COMPRESSION_ITERATIONS)
+
+
+def pack(document, block_size=BLOCK_SIZE, progress=None):
+    blocks, starts, _ = token_blocks(document, block_size)
     packed, descriptors = bytearray(), []
-    for first_verse, block in zip(starts, blocks):
-        compressed = raw_deflate_compress(block, numiterations=COMPRESSION_ITERATIONS)
+    for ordinal, (first_verse, block) in enumerate(zip(starts, blocks), 1):
+        compressed = compress_block(block)
         descriptors.append((len(packed), len(compressed), len(block), first_verse))
         packed.extend(compressed)
+        if progress:
+            progress(ordinal, len(blocks), len(packed))
     return bytes(packed), descriptors
 
 
@@ -90,8 +148,14 @@ def block_index_layout(blocks):
     return ("uint16_t", 8) if max(block[0] for block in blocks) <= 0xFFFF else ("uint32_t", 12)
 
 
-def render(document):
-    packed, blocks = pack(document)
+def render(document, progress=None):
+    packed, blocks = pack(document, progress=progress)
+    words = token_dictionary(source_verses(document))
+    dictionary = b"".join(words)
+    word_offsets, offset = [0], 0
+    for word in words:
+        offset += len(word)
+        word_offsets.append(offset)
     offset_type, index_bytes = block_index_layout(blocks)
     digest = hashlib.sha256(json.dumps(document, ensure_ascii=False, sort_keys=True,
                                      separators=(",", ":")).encode()).hexdigest()
@@ -99,23 +163,34 @@ def render(document):
              f"// Compressed with Google Zopfli {ZOPFLI_VERSION} ({COMPRESSION_ITERATIONS} iterations).",
              "// Offline Reader corpus. Public domain; see tools/bible/README.md.",
              "// Firmware typography: ASCII quotes, apostrophes, dashes and spaces; wording unchanged.",
+             "// Exact word tokens expand only the selected verse in the same 4 KiB scratch.",
              f"// Canonical source SHA-256: {digest}",
              "#pragma once", '#include "ReaderLookup.h"',
              "namespace mesh { namespace bible { namespace generated {",
              f'static_assert(kBlockSize == {BLOCK_SIZE}, "Regenerate Reader data after changing block size");',
              f"using ReaderBlockIndex = BlockIndexT<{offset_type}>;",
-             f"using ReaderCorpus = CorpusT<{offset_type}>;",
+             f"using ReaderCorpus = {'TokenCorpusT' if words else 'CorpusT'}<{offset_type}>;",
              f'static_assert(sizeof(ReaderBlockIndex) == {index_bytes}, "Reader flash-size accounting expects a {index_bytes}-byte index");',
              "static const uint8_t readerData[] = {"]
     for start in range(0, len(packed), 20):
         lines.append("  " + ",".join(f"0x{v:02x}" for v in packed[start:start + 20]) + ",")
     lines += ["};", "static const ReaderBlockIndex readerBlocks[] = {"]
     lines += ["  {%d, %d, %d, %d}," % block for block in blocks]
-    lines += ["};", "static const ReaderCorpus readerCorpus = {",
+    lines += ["};"]
+    if words:
+        lines += ["static const uint8_t readerWords[] = {"]
+        for start in range(0, len(dictionary), 20):
+            lines.append("  " + ",".join(f"0x{v:02x}" for v in dictionary[start:start + 20]) + ",")
+        lines += ["};", "static const uint16_t readerWordOffsets[] = {",
+                  "  " + ",".join(map(str, word_offsets)), "};"]
+    lines += ["static const ReaderCorpus readerCorpus = {",
               f"  {json.dumps(document['translation'], ensure_ascii=False)},",
               f"  {json.dumps(document['attribution'], ensure_ascii=False)},",
               "  readerData, sizeof(readerData), readerBlocks,",
-              "  sizeof(readerBlocks) / sizeof(readerBlocks[0]), kVerseCount", "};", "}}}", ""]
+              "  sizeof(readerBlocks) / sizeof(readerBlocks[0]), kVerseCount" + ("," if words else "")]
+    if words:
+        lines += [f"  readerWords, sizeof(readerWords), readerWordOffsets, {len(words)}"]
+    lines += ["};", "}}}", ""]
     return "\n".join(lines)
 
 
@@ -125,19 +200,25 @@ def main():
     parser.add_argument("--output", type=Path, default=HEADER)
     parser.add_argument("--check", action="store_true")
     parser.add_argument("--compare", action="store_true", help="compare 1/2/4 KiB blocks without writing")
+    parser.add_argument("--progress", action="store_true", help="report canonical block-compression progress")
     args = parser.parse_args()
     document = json.loads(args.source.read_text(encoding="utf-8"))
     if args.compare:
         for size in (1024, 2048, 4096):
             packed, blocks = pack(document, size)
             tables = len(blocks) * block_index_layout(blocks)[1]
-            plain = sum(block[2] for block in blocks)
+            verses = source_verses(document, size)
+            dictionary = dictionary_storage(token_dictionary(verses))
+            plain = sum(len(verse) + 1 for verse in verses)
+            total = len(packed) + tables + dictionary
             print(f"{size} byte blocks: {len(blocks)} blocks, {plain} plain, "
-                  f"{len(packed)} compressed, {tables} index, "
-                  f"{len(packed) + tables} total, "
-                  f"{100 * (1 - (len(packed) + tables) / plain):.2f}% saved")
+                  f"{len(packed)} compressed, {tables} index, {dictionary} dictionary, "
+                  f"{total} total, {100 * (1 - total / plain):.2f}% saved")
         return
-    header = render(document)
+    progress = (lambda done, total, size: print(
+        f"Reader block {done}/{total}: {size} compressed bytes", file=sys.stderr, flush=True)
+        ) if args.progress else None
+    header = render(document, progress=progress)
     if args.check:
         if args.output.read_text(encoding="utf-8") != header:
             raise SystemExit("Reader header differs; regenerate with tools/bible/pack_reader.py")
@@ -145,11 +226,14 @@ def main():
         args.output.parent.mkdir(parents=True, exist_ok=True)
         args.output.write_text(header, encoding="utf-8", newline="\n")
     packed, blocks = pack(document)
-    plain = sum(block[2] for block in blocks)
+    verses = source_verses(document)
+    plain = sum(len(verse) + 1 for verse in verses)
     tables = len(blocks) * block_index_layout(blocks)[1]
+    dictionary = dictionary_storage(token_dictionary(verses))
     print(f"Reader: {len(references())} verses, {len(blocks)} blocks; "
-          f"{plain} plain bytes -> {len(packed)} DEFLATE + {tables} index bytes "
-          f"({100 * (1 - (len(packed) + tables) / plain):.1f}% smaller incl. indexes)")
+          f"{plain} plain bytes -> {len(packed)} DEFLATE + {tables} index "
+          f"+ {dictionary} dictionary bytes "
+          f"({100 * (1 - (len(packed) + tables + dictionary) / plain):.1f}% smaller incl. tables)")
 
 
 if __name__ == "__main__":