Fit direct OTA administration into RAK4631 Full
This commit is contained in:
1 parent
9d2f7a3e27
commit
7d413a6806
21 files changed
+1926
-61
No files matched your search
@@ -321,6 +321,14 @@ independent blocks. Native tests compare every verse and output byte, exercise
|
||||
malformed inputs, and check the actual ARM stack against the existing 8 KiB
|
||||
loop stack. Linked-ELF tests retain the same text and reply checks.
|
||||
|
||||
RAK4631 Full opts into `COMPANION_READER_COMPACT_STREAM=1` and a separate
|
||||
single-stream asset. Its data and vocabulary occupy 28,126 bytes, saving another
|
||||
661 bytes while retaining every verse, font and the same 4 KiB scratch.
|
||||
The tradeoff is a cold lookup that may decode up to 50,732 encoded bytes before
|
||||
the selected verse. Other nRF52 boards retain the three-stream asset and its
|
||||
shorter cold lookup. The single stream uses the existing unframed format;
|
||||
larger custom corpora still use bounded, independently framed streams.
|
||||
|
||||
`reader-stream-tokens.json` is the pinned compression vocabulary. It substitutes
|
||||
only exact source byte strings, including substrings of words. Host-only Google
|
||||
Zopfli uses 1,000 iterations with a 4 KiB match window; its Apache-2.0 source,
|
||||
@@ -330,5 +338,6 @@ committed header and require no extra downloads.
|
||||
|
||||
```sh
|
||||
python tools/bible/pack_reader_stream.py --check
|
||||
python tools/bible/pack_reader_stream.py --compact --check
|
||||
python test/test_reader_stream.py
|
||||
```
|
||||
@@ -14,9 +14,11 @@ import zlib
|
||||
import pack_reader as blocks
|
||||
|
||||
HEADER = blocks.ROOT / "src/helpers/bible/ReaderStreamData.generated.h"
|
||||
COMPACT_HEADER = blocks.ROOT / "src/helpers/bible/ReaderCompactStreamData.generated.h"
|
||||
TOKENS = Path(__file__).with_name("reader-stream-tokens.json")
|
||||
COMPRESSOR = Path(__file__).with_name("zopfli4k")
|
||||
STREAM_LIMIT = 17000
|
||||
COMPACT_STREAM_LIMIT = 60000
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
@@ -55,7 +57,7 @@ def token_dictionary(verses):
|
||||
return [word for word in words if any(word in verse for verse in verses)]
|
||||
|
||||
|
||||
def encode(document):
|
||||
def encode(document, *, compact=False):
|
||||
verses = blocks.source_verses(document)
|
||||
words = token_dictionary(verses)
|
||||
tokens = {word: bytes([128 + index]) for index, word in enumerate(words)}
|
||||
@@ -72,13 +74,15 @@ def encode(document):
|
||||
records = [verse + b"\0" for verse in encoded_verses]
|
||||
chunks, first, current = [], 0, b""
|
||||
for index, record in enumerate(records):
|
||||
if current and len(current) + len(record) > STREAM_LIMIT:
|
||||
if current and len(current) + len(record) > (COMPACT_STREAM_LIMIT if compact else STREAM_LIMIT):
|
||||
chunks.append((first, current))
|
||||
first, current = index, b""
|
||||
current += record
|
||||
chunks.append((first, current))
|
||||
# Independent 12-bit histories bound cold lookup work without adding a
|
||||
# buffer. Headers keep both compressed and decoded lengths bounded.
|
||||
# A single stream needs no framing. Larger corpora retain independently
|
||||
# bounded 12-bit histories and six-byte headers, without another buffer.
|
||||
if compact and len(chunks) == 1:
|
||||
return compress_4k(chunks[0][1]), b"".join(records), words, 0
|
||||
packed = bytearray()
|
||||
for first, plain in chunks:
|
||||
compressed = compress_4k(plain)
|
||||
@@ -87,13 +91,14 @@ def encode(document):
|
||||
return bytes(packed), b"".join(records), words, len(chunks)
|
||||
|
||||
|
||||
def render(document):
|
||||
packed, plain, words, chunk_count = encode(document)
|
||||
def render(document, *, compact=False):
|
||||
packed, plain, words, chunk_count = encode(document, compact=compact)
|
||||
dictionary_plain = b"\0".join(words) + b"\0" if words else b""
|
||||
dictionary = compress_4k(dictionary_plain) if words else b""
|
||||
digest = hashlib.sha256(json.dumps(document, ensure_ascii=False, sort_keys=True,
|
||||
separators=(",", ":")).encode()).hexdigest()
|
||||
lines = ["// Generated by tools/bible/pack_reader_stream.py; do not edit.",
|
||||
command = "tools/bible/pack_reader_stream.py" + (" --compact" if compact else "")
|
||||
lines = [f"// Generated by {command}; do not edit.",
|
||||
"// Offline Reader corpus. Public domain; see tools/bible/README.md.",
|
||||
"// Google Zopfli, 1000 iterations, 4096-byte history; exact ASCII text retained.",
|
||||
f"// Canonical source SHA-256: {digest}",
|
||||
@@ -120,22 +125,24 @@ def render(document):
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--source", type=Path, default=blocks.SOURCE)
|
||||
parser.add_argument("--output", type=Path, default=HEADER)
|
||||
parser.add_argument("--output", type=Path)
|
||||
parser.add_argument("--compact", action="store_true", help="RAK4631 Full: smaller asset with slower cold lookup")
|
||||
parser.add_argument("--check", action="store_true")
|
||||
args = parser.parse_args()
|
||||
document = json.loads(args.source.read_text(encoding="utf-8"))
|
||||
header = render(document)
|
||||
output = args.output or (COMPACT_HEADER if args.compact else HEADER)
|
||||
header = render(document, compact=args.compact)
|
||||
if args.check:
|
||||
if args.output.read_text(encoding="ascii") != header:
|
||||
if output.read_text(encoding="ascii") != header:
|
||||
raise SystemExit("Reader stream header differs; regenerate with tools/bible/pack_reader_stream.py")
|
||||
else:
|
||||
args.output.write_text(header, encoding="ascii", newline="\n")
|
||||
packed, plain, words, chunk_count = encode(document)
|
||||
output.write_text(header, encoding="ascii", newline="\n")
|
||||
packed, plain, words, chunk_count = encode(document, compact=args.compact)
|
||||
dictionary_plain = b"\0".join(words) + b"\0" if words else b""
|
||||
dictionary_size = len(compress_4k(dictionary_plain)) if words else 0
|
||||
print(f"Reader stream: {len(blocks.references())} unchanged verses, "
|
||||
f"{len(plain)} encoded bytes -> {len(packed)} DEFLATE + "
|
||||
f"{dictionary_size} dictionary bytes; {chunk_count} independent streams, 4096-byte history")
|
||||
f"{dictionary_size} dictionary bytes; {chunk_count or 1} independent streams, 4096-byte history")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -374,8 +374,7 @@ class CompanionTransport:
|
||||
await self._guard_mode(key, 1, bytes([contact["type"]]))
|
||||
except CompanionError as exc:
|
||||
raise CompanionError(
|
||||
"The USB radio needs current Full Companion firmware with the zero-hop OTA admin guard. "
|
||||
"RAK4631 Full cannot fit this helper; use an ESP32 Full Companion or USB Repeater.") from exc
|
||||
"The USB radio needs current Full Companion firmware with the version 2 zero-hop OTA admin guard.") from exc
|
||||
self._guarded_key = key
|
||||
if key not in self._authenticated:
|
||||
# The library's send_login_sync accepts any login-success
|
||||
|
||||
Reference in new issue
Block a user