diff --git a/examples/companion_radio/CompanionHelpWriter.h b/examples/companion_radio/CompanionHelpWriter.h index 5299e28e4..dfac47e77 100644 --- a/examples/companion_radio/CompanionHelpWriter.h +++ b/examples/companion_radio/CompanionHelpWriter.h @@ -30,42 +30,59 @@ size_t writeHelpChunks(Output& output, const char* text, size_t length) { #if __cplusplus >= 201703L namespace help_packing { -// Help is ASCII. Bytes 0x80..0x8f encode these common fragments; the original -// text exists only during constant evaluation, never alongside the packed data. -constexpr size_t TokenCount = 16; -constexpr char Dictionary[] = - " get/set \0" " get \0" " set \0" "get/set \0" - "bluetooth.\0" "pwrmgt.\0" "radio.\0" "default\0" - "preamble\0" "on|off\0" " (saved; \0" " \0" - "get \0" "set \0" "\r\n\0" " / \0"; +// ASCII literals retain bit 7 clear. Two-byte references encode a 3..18 byte +// match at distance 1..2048. Only the final packed bytes survive evaluation; +// the hash chains and original text occupy no runtime storage. +constexpr size_t WindowSize = 2048; +constexpr size_t MaximumMatch = 18; +constexpr uint8_t hash(const char* text) { + return static_cast((static_cast(text[0]) * 31u + + static_cast(text[1])) * 31u + static_cast(text[2])); +} -struct TokenOffsets { uint8_t value[TokenCount + 1] = {}; }; -constexpr TokenOffsets tokenOffsets() { - TokenOffsets result{}; - size_t offset = 0; - for (size_t token = 0; token < TokenCount; ++token) { - result.value[token] = static_cast(offset); - while (Dictionary[offset]) ++offset; - ++offset; +template +constexpr size_t encode(const char (&text)[N], Emit emit) { + static_assert(N <= 65535, "Compile-time help offsets use 16 bits"); + uint16_t heads[256] = {}, previous[N] = {}; + size_t position = 0, packed = 0; + while (position + 1 < N) { + const size_t remaining = N - position - 1; + const size_t maximum = remaining < MaximumMatch ? remaining : MaximumMatch; + size_t best = 0, distance = 0; + if (remaining >= 3) { + uint16_t link = heads[hash(text + position)]; + while (link) { + const size_t candidate = link - 1; + const size_t back = position - candidate; + if (back > WindowSize) break; + size_t match = 0; + while (match < maximum && text[candidate + match] == text[position + match]) ++match; + if (match > best) { + best = match; + distance = back; + if (best == maximum) break; + } + link = previous[candidate]; + } + } + const size_t step = best >= 3 ? best : 1; + if (best >= 3) { + emit(static_cast(0x80 | ((best - 3) << 3) | ((distance - 1) >> 8))); + emit(static_cast(distance - 1)); + packed += 2; + } else { + emit(static_cast(text[position])); + ++packed; + } + // Index skipped match bytes too, preserving greedy longest-match choices. + for (size_t end = position + step; position < end; ++position) { + if (position + 3 >= N) continue; + const uint8_t bucket = hash(text + position); + previous[position] = heads[bucket]; + heads[bucket] = static_cast(position + 1); + } } - result.value[TokenCount] = static_cast(offset); - return result; -} -static_assert(sizeof(Dictionary) < 256, "Help dictionary offsets need one byte"); -constexpr TokenOffsets Offsets = tokenOffsets(); -constexpr size_t tokenLength(size_t token) { - return Offsets.value[token + 1] - Offsets.value[token] - 1; -} -constexpr size_t matchToken(const char* text, size_t remaining) { - for (size_t token = 0; token < TokenCount; ++token) { - const char* fragment = Dictionary + Offsets.value[token]; - const size_t length = tokenLength(token); - if (length > remaining || *text != *fragment) continue; - size_t i = 1; - while (i < length && text[i] == fragment[i]) ++i; - if (i == length) return token; - } - return TokenCount; + return packed; } } // namespace help_packing @@ -81,12 +98,7 @@ constexpr bool validHelpText(const char (&text)[N]) { template constexpr size_t packedHelpSize(const char (&text)[N]) { - size_t packed = 0; - for (size_t i = 0; i + 1 < N; ++packed) { - const size_t token = help_packing::matchToken(text + i, N - i - 1); - i += token < help_packing::TokenCount ? help_packing::tokenLength(token) : 1; - } - return packed; + return help_packing::encode(text, [](uint8_t) constexpr {}); } template struct PackedHelp { @@ -98,41 +110,36 @@ template constexpr PackedHelp packHelp(const char (&text)[N]) { PackedHelp result{}; size_t packed = 0; - for (size_t i = 0; i + 1 < N;) { - const size_t token = help_packing::matchToken(text + i, N - i - 1); - if (token < help_packing::TokenCount) { - result.bytes[packed++] = static_cast(0x80 + token); - i += help_packing::tokenLength(token); - } else { - result.bytes[packed++] = static_cast(text[i++]); - } - } + help_packing::encode(text, [&](uint8_t byte) constexpr { result.bytes[packed++] = byte; }); return result; } template size_t writePackedHelp(Output& output, const uint8_t* packed, size_t length) { if (!packed) return 0; - size_t position = 0, written = 0; - const char* expansion = nullptr; - while (position < length || expansion) { + uint8_t history[help_packing::WindowSize]; + size_t position = 0, produced = 0, written = 0; + size_t distance = 0, remaining = 0; + while (position < length || remaining) { char chunk[64]; size_t used = 0; - while (used < sizeof(chunk) && (position < length || expansion)) { - if (expansion) { - const char next = *expansion++; - if (next) chunk[used++] = next; - else expansion = nullptr; + while (used < sizeof(chunk) && (position < length || remaining)) { + uint8_t next; + if (remaining) { + next = history[(produced - distance) & (help_packing::WindowSize - 1)]; + --remaining; } else { - const uint8_t next = packed[position++]; - if (next >= 0x80) { - const size_t token = next - 0x80; - if (token >= help_packing::TokenCount) return written; - expansion = help_packing::Dictionary + help_packing::Offsets.value[token]; - } else { - chunk[used++] = static_cast(next); + next = packed[position++]; + if (next & 0x80) { + if (position == length) return written; + distance = ((static_cast(next & 7) << 8) | packed[position++]) + 1; + remaining = ((next >> 3) & 15) + 3; + if (distance > produced) return written; + continue; } } + history[produced++ & (help_packing::WindowSize - 1)] = next; + chunk[used++] = static_cast(next); } const size_t accepted = writeHelpChunks(output, chunk, used); written += accepted; diff --git a/test/test_companion_help_writer.py b/test/test_companion_help_writer.py index 893884010..adbeb88f3 100644 --- a/test/test_companion_help_writer.py +++ b/test/test_companion_help_writer.py @@ -114,7 +114,7 @@ int main() { mesh::companion::packedHelpSize(literal)>(literal); static_assert(sizeof(packed) == mesh::companion::packedHelpSize(literal), "No raw-size padding retained"); - static_assert(sizeof(packed) + 40 < sizeof(literal), "Fragments compress"); + static_assert(sizeof(packed) < sizeof(literal), "Repeated fragments compress"); Output decoded; assert(mesh::companion::writePackedHelp(decoded,packed.bytes,sizeof(packed))==sizeof(literal)-1); assert(decoded.text==literal); @@ -135,6 +135,41 @@ int main() { assert(mesh::companion::writePackedHelp(unused,packed.bytes,0)==0 && unused.writes.empty()); const uint8_t invalid_token[]={0xff}; assert(mesh::companion::writePackedHelp(unused,invalid_token,1)==0 && unused.writes.empty()); + + // A match may overlap itself: one literal followed by distance-one runs. + constexpr char overlap[]="aaaaaaaaaaaaaaaaaaaaaaa"; + static constexpr auto overlap_packed=mesh::companion::packHelp< + mesh::companion::packedHelpSize(overlap)>(overlap); + static_assert(sizeof(overlap_packed)==5 && overlap_packed.bytes[0]=='a' && + overlap_packed.bytes[1]==0xf8 && overlap_packed.bytes[2]==0 && + overlap_packed.bytes[3]==0x88 && overlap_packed.bytes[4]==0, + "Greedy longest match and overlapping references"); + Output repeated; + assert(mesh::companion::writePackedHelp(repeated,overlap_packed.bytes, + sizeof(overlap_packed))==sizeof(overlap)-1 && repeated.text==overlap); + + // Exercise both the 2048-byte distance limit and the ring wrap, independent + // of the encoder's choice of nearer repeated phrases. + std::vector boundary; + for(size_t i=0;i<2048;++i) boundary.push_back('!'+(i%90)); + std::string expected(boundary.begin(),boundary.end()); + boundary.push_back(0xff);boundary.push_back(0xff); // length18, distance2048 + expected+=expected.substr(0,18); + boundary.push_back(0xf8);boundary.push_back(0); // length18, distance1 + expected+=std::string(18,expected.back()); + Output wrapped;wrapped.throttle=5; + assert(mesh::companion::writePackedHelp(wrapped,boundary.data(),boundary.size())==expected.size()); + assert(wrapped.text==expected); + for(const auto& corrupt : std::vector>{ + {0x80}, {0x80,0}, {0xff,0xff}, {'x',0x80,1}}) { + Output rejected; + assert(mesh::companion::writePackedHelp(rejected,corrupt.data(),corrupt.size())==0); + assert(rejected.text.empty() && rejected.writes.empty()); + } + std::vector late_bad(64,'q');late_bad.push_back(0x80); + Output prefix; + assert(mesh::companion::writePackedHelp(prefix,late_bad.data(),late_bad.size())==64); + assert(prefix.text==std::string(64,'q') && prefix.writes.size()==1); } '''