Store complete RAK help with bounded LZSS compression

This commit is contained in:
mikecarper committed 2026-10-23 10:02:01 -07:00
1 parent 629db8542d
commit 0cdf6f9aa6
2 files changed
+107 -65

No files matched your search

+71 -64
View File
@@ -30,42 +30,59 @@ size_t writeHelpChunks(Output& output, const char* text, size_t length) {
#if __cplusplus >= 201703L
namespace help_packing {
// Help is ASCII. Bytes 0x80..0x8f encode these common fragments; the original
// text exists only during constant evaluation, never alongside the packed data.
constexpr size_t TokenCount = 16;
constexpr char Dictionary[] =
" get/set \0" " get \0" " set \0" "get/set \0"
"bluetooth.\0" "pwrmgt.\0" "radio.\0" "default\0"
"preamble\0" "on|off\0" " (saved; \0" " \0"
"get \0" "set \0" "\r\n\0" " / \0";
// ASCII literals retain bit 7 clear. Two-byte references encode a 3..18 byte
// match at distance 1..2048. Only the final packed bytes survive evaluation;
// the hash chains and original text occupy no runtime storage.
constexpr size_t WindowSize = 2048;
constexpr size_t MaximumMatch = 18;
constexpr uint8_t hash(const char* text) {
return static_cast<uint8_t>((static_cast<uint8_t>(text[0]) * 31u +
static_cast<uint8_t>(text[1])) * 31u + static_cast<uint8_t>(text[2]));
}
struct TokenOffsets { uint8_t value[TokenCount + 1] = {}; };
constexpr TokenOffsets tokenOffsets() {
TokenOffsets result{};
size_t offset = 0;
for (size_t token = 0; token < TokenCount; ++token) {
result.value[token] = static_cast<uint8_t>(offset);
while (Dictionary[offset]) ++offset;
++offset;
template<size_t N, typename Emit>
constexpr size_t encode(const char (&text)[N], Emit emit) {
static_assert(N <= 65535, "Compile-time help offsets use 16 bits");
uint16_t heads[256] = {}, previous[N] = {};
size_t position = 0, packed = 0;
while (position + 1 < N) {
const size_t remaining = N - position - 1;
const size_t maximum = remaining < MaximumMatch ? remaining : MaximumMatch;
size_t best = 0, distance = 0;
if (remaining >= 3) {
uint16_t link = heads[hash(text + position)];
while (link) {
const size_t candidate = link - 1;
const size_t back = position - candidate;
if (back > WindowSize) break;
size_t match = 0;
while (match < maximum && text[candidate + match] == text[position + match]) ++match;
if (match > best) {
best = match;
distance = back;
if (best == maximum) break;
}
link = previous[candidate];
}
}
const size_t step = best >= 3 ? best : 1;
if (best >= 3) {
emit(static_cast<uint8_t>(0x80 | ((best - 3) << 3) | ((distance - 1) >> 8)));
emit(static_cast<uint8_t>(distance - 1));
packed += 2;
} else {
emit(static_cast<uint8_t>(text[position]));
++packed;
}
// Index skipped match bytes too, preserving greedy longest-match choices.
for (size_t end = position + step; position < end; ++position) {
if (position + 3 >= N) continue;
const uint8_t bucket = hash(text + position);
previous[position] = heads[bucket];
heads[bucket] = static_cast<uint16_t>(position + 1);
}
}
result.value[TokenCount] = static_cast<uint8_t>(offset);
return result;
}
static_assert(sizeof(Dictionary) < 256, "Help dictionary offsets need one byte");
constexpr TokenOffsets Offsets = tokenOffsets();
constexpr size_t tokenLength(size_t token) {
return Offsets.value[token + 1] - Offsets.value[token] - 1;
}
constexpr size_t matchToken(const char* text, size_t remaining) {
for (size_t token = 0; token < TokenCount; ++token) {
const char* fragment = Dictionary + Offsets.value[token];
const size_t length = tokenLength(token);
if (length > remaining || *text != *fragment) continue;
size_t i = 1;
while (i < length && text[i] == fragment[i]) ++i;
if (i == length) return token;
}
return TokenCount;
return packed;
}
} // namespace help_packing
@@ -81,12 +98,7 @@ constexpr bool validHelpText(const char (&text)[N]) {
template<size_t N>
constexpr size_t packedHelpSize(const char (&text)[N]) {
size_t packed = 0;
for (size_t i = 0; i + 1 < N; ++packed) {
const size_t token = help_packing::matchToken(text + i, N - i - 1);
i += token < help_packing::TokenCount ? help_packing::tokenLength(token) : 1;
}
return packed;
return help_packing::encode(text, [](uint8_t) constexpr {});
}
template<size_t N> struct PackedHelp {
@@ -98,41 +110,36 @@ template<size_t PackedSize, size_t N>
constexpr PackedHelp<PackedSize> packHelp(const char (&text)[N]) {
PackedHelp<PackedSize> result{};
size_t packed = 0;
for (size_t i = 0; i + 1 < N;) {
const size_t token = help_packing::matchToken(text + i, N - i - 1);
if (token < help_packing::TokenCount) {
result.bytes[packed++] = static_cast<uint8_t>(0x80 + token);
i += help_packing::tokenLength(token);
} else {
result.bytes[packed++] = static_cast<uint8_t>(text[i++]);
}
}
help_packing::encode(text, [&](uint8_t byte) constexpr { result.bytes[packed++] = byte; });
return result;
}
template<typename Output>
size_t writePackedHelp(Output& output, const uint8_t* packed, size_t length) {
if (!packed) return 0;
size_t position = 0, written = 0;
const char* expansion = nullptr;
while (position < length || expansion) {
uint8_t history[help_packing::WindowSize];
size_t position = 0, produced = 0, written = 0;
size_t distance = 0, remaining = 0;
while (position < length || remaining) {
char chunk[64];
size_t used = 0;
while (used < sizeof(chunk) && (position < length || expansion)) {
if (expansion) {
const char next = *expansion++;
if (next) chunk[used++] = next;
else expansion = nullptr;
while (used < sizeof(chunk) && (position < length || remaining)) {
uint8_t next;
if (remaining) {
next = history[(produced - distance) & (help_packing::WindowSize - 1)];
--remaining;
} else {
const uint8_t next = packed[position++];
if (next >= 0x80) {
const size_t token = next - 0x80;
if (token >= help_packing::TokenCount) return written;
expansion = help_packing::Dictionary + help_packing::Offsets.value[token];
} else {
chunk[used++] = static_cast<char>(next);
next = packed[position++];
if (next & 0x80) {
if (position == length) return written;
distance = ((static_cast<size_t>(next & 7) << 8) | packed[position++]) + 1;
remaining = ((next >> 3) & 15) + 3;
if (distance > produced) return written;
continue;
}
}
history[produced++ & (help_packing::WindowSize - 1)] = next;
chunk[used++] = static_cast<char>(next);
}
const size_t accepted = writeHelpChunks(output, chunk, used);
written += accepted;
+36 -1
View File
@@ -114,7 +114,7 @@ int main() {
mesh::companion::packedHelpSize(literal)>(literal);
static_assert(sizeof(packed) == mesh::companion::packedHelpSize(literal),
"No raw-size padding retained");
static_assert(sizeof(packed) + 40 < sizeof(literal), "Fragments compress");
static_assert(sizeof(packed) < sizeof(literal), "Repeated fragments compress");
Output decoded;
assert(mesh::companion::writePackedHelp(decoded,packed.bytes,sizeof(packed))==sizeof(literal)-1);
assert(decoded.text==literal);
@@ -135,6 +135,41 @@ int main() {
assert(mesh::companion::writePackedHelp(unused,packed.bytes,0)==0 && unused.writes.empty());
const uint8_t invalid_token[]={0xff};
assert(mesh::companion::writePackedHelp(unused,invalid_token,1)==0 && unused.writes.empty());
// A match may overlap itself: one literal followed by distance-one runs.
constexpr char overlap[]="aaaaaaaaaaaaaaaaaaaaaaa";
static constexpr auto overlap_packed=mesh::companion::packHelp<
mesh::companion::packedHelpSize(overlap)>(overlap);
static_assert(sizeof(overlap_packed)==5 && overlap_packed.bytes[0]=='a' &&
overlap_packed.bytes[1]==0xf8 && overlap_packed.bytes[2]==0 &&
overlap_packed.bytes[3]==0x88 && overlap_packed.bytes[4]==0,
"Greedy longest match and overlapping references");
Output repeated;
assert(mesh::companion::writePackedHelp(repeated,overlap_packed.bytes,
sizeof(overlap_packed))==sizeof(overlap)-1 && repeated.text==overlap);
// Exercise both the 2048-byte distance limit and the ring wrap, independent
// of the encoder's choice of nearer repeated phrases.
std::vector<uint8_t> boundary;
for(size_t i=0;i<2048;++i) boundary.push_back('!'+(i%90));
std::string expected(boundary.begin(),boundary.end());
boundary.push_back(0xff);boundary.push_back(0xff); // length18, distance2048
expected+=expected.substr(0,18);
boundary.push_back(0xf8);boundary.push_back(0); // length18, distance1
expected+=std::string(18,expected.back());
Output wrapped;wrapped.throttle=5;
assert(mesh::companion::writePackedHelp(wrapped,boundary.data(),boundary.size())==expected.size());
assert(wrapped.text==expected);
for(const auto& corrupt : std::vector<std::vector<uint8_t>>{
{0x80}, {0x80,0}, {0xff,0xff}, {'x',0x80,1}}) {
Output rejected;
assert(mesh::companion::writePackedHelp(rejected,corrupt.data(),corrupt.size())==0);
assert(rejected.text.empty() && rejected.writes.empty());
}
std::vector<uint8_t> late_bad(64,'q');late_bad.push_back(0x80);
Output prefix;
assert(mesh::companion::writePackedHelp(prefix,late_bad.data(),late_bad.size())==64);
assert(prefix.text==std::string(64,'q') && prefix.writes.size()==1);
}
'''