Reduce contact RAM across boards and compact Full firmware

This commit is contained in:
mikecarper committed 2026-10-19 01:55:25 -07:00
1 parent 8d4264e272
commit 036661e7e4
19 files changed
+2355 -2030

No files matched your search

+2 -2
View File
@@ -92,9 +92,9 @@ class BaseChatMesh {
public:
#if defined(ESP32_PLATFORM) && defined(BOARD_HAS_PSRAM)
ContactInfo contacts_fallback[MAX_ANON_CONTACTS + 16];
int sort_array_fallback[MAX_ANON_CONTACTS + 16] = {};
ContactIndex sort_array_fallback[MAX_ANON_CONTACTS + 16] = {};
ContactInfo* contacts = contacts_fallback;
int* sort_array = sort_array_fallback;
ContactIndex* sort_array = sort_array_fallback;
int contact_capacity = MAX_ANON_CONTACTS + 16;
#else
ContactInfo contacts[MAX_ANON_CONTACTS + MAX_CONTACTS];
+25 -5
View File
@@ -8,6 +8,22 @@
using namespace mesh::bible;
template <typename Offset>
static void checkOffsetWidth(uint32_t offset) {
static const uint8_t stored[] = {0x01, 0x02, 0x00, 0xfd, 0xff, 'x', 0x00};
std::vector<uint8_t> data(offset + sizeof(stored));
memcpy(data.data() + offset, stored, sizeof(stored));
const BlockIndexT<Offset> index = {static_cast<Offset>(offset), sizeof(stored), 2, 0};
CorpusT<Offset> corpus = {"test", "test", data.data(), uint32_t(data.size()), &index, 1, kVerseCount};
char scratch[kBlockSize];
const char* text;
assert(lookup(corpus, {1, 1}, scratch, sizeof(scratch), text) == LookupResult::Found);
assert(strcmp(text, "x") == 0);
corpus.data_size--;
assert(lookup(corpus, {1, 1}, scratch, sizeof(scratch), text) == LookupResult::Corrupt);
assert(text == nullptr);
}
static void drain(Stream& output) {
unsigned calls = 0;
while (mesh::readerCommandActive()) {
@@ -70,6 +86,10 @@ static void checkSelection(unsigned chapter, unsigned first, unsigned last, bool
}
int main(int argc, char**) {
static_assert(sizeof(BlockIndexT<uint16_t>) == 8, "halfword index alignment");
static_assert(sizeof(BlockIndexT<uint32_t>) == 12, "wide index alignment");
checkOffsetWidth<uint16_t>(UINT16_MAX);
checkOffsetWidth<uint32_t>(uint32_t(UINT16_MAX) + 1);
Reference ref = {};
for (const char* cmd : {"get reader 3:16", " GET\treader 3:16 \t", "get reader 03:16"}) {
assert(parse(cmd, ref) == ParseResult::Valid && ref.chapter == 3 && ref.verse == 16);
@@ -87,7 +107,7 @@ int main(int argc, char**) {
assert(!mesh::handleReaderCommand(cmd, output) && output.text.empty());
}
assert(parse(nullptr, ref) == ParseResult::NoMatch);
const Corpus& corpus = generated::readerCorpus;
const auto& corpus = generated::readerCorpus;
struct Guarded { uint32_t pre; char scratch[kBlockSize]; uint32_t post; } buf = {};
buf.pre = 0x11223344; buf.post = 0xaabbccdd;
unsigned count = 0;
@@ -189,7 +209,7 @@ int main(int argc, char**) {
const char* text;
assert(lookup(corpus, {22, 1}, buf.scratch, kBlockSize, text) == LookupResult::Invalid);
assert(lookup(corpus, {1, 1}, buf.scratch, kBlockSize - 1, text) == LookupResult::Corrupt);
Corpus bad = corpus;
auto bad = corpus;
bad.data_size = 0;
assert(lookup(bad, {1, 1}, buf.scratch, kBlockSize, text) == LookupResult::Corrupt);
bad = corpus; bad.block_count = 0;
@@ -200,7 +220,7 @@ int main(int argc, char**) {
// One-byte stored DEFLATE block: distinguish an absent verse marker from
// damaged text without a terminator, and never return an unsafe pointer.
uint8_t stored[] = {0x01, 0x01, 0x00, 0xfe, 0xff, 0x00};
const BlockIndex stored_index = {0, sizeof(stored), 1, 0};
const generated::ReaderBlockIndex stored_index = {0, sizeof(stored), 1, 0};
bad = corpus; bad.data = stored; bad.data_size = sizeof(stored);
bad.blocks = &stored_index; bad.block_count = 1;
assert(lookup(bad, {1, 1}, buf.scratch, kBlockSize, text) == LookupResult::Missing);
@@ -209,7 +229,7 @@ int main(int argc, char**) {
assert(lookup(bad, {1, 1}, buf.scratch, kBlockSize, text) == LookupResult::Corrupt);
assert(text == nullptr);
assert(lookup(bad, {1, 2}, buf.scratch, kBlockSize, text) == LookupResult::Corrupt);
std::vector<BlockIndex> blocks(corpus.blocks, corpus.blocks + corpus.block_count);
std::vector<generated::ReaderBlockIndex> blocks(corpus.blocks, corpus.blocks + corpus.block_count);
bad = corpus; bad.blocks = blocks.data();
for (int mutation = 0; mutation != 5; ++mutation) {
blocks[0] = corpus.blocks[0];
@@ -217,7 +237,7 @@ int main(int argc, char**) {
case 0: blocks[0].plain = kBlockSize + 1; break;
case 1: blocks[0].packed--; break;
case 2: blocks[0].packed++; break; // Trailing bytes are rejected.
case 3: blocks[0].offset = UINT32_MAX; break;
case 3: blocks[0].offset = static_cast<decltype(blocks[0].offset)>(-1); break;
case 4: blocks[0].first_verse = 1; break;
}
assert(lookup(bad, {1, 1}, buf.scratch, kBlockSize, text) == LookupResult::Corrupt);
+1 -1
View File
@@ -500,7 +500,7 @@ static void raw_read_recheck_preserves_old_and_new_unread_page_scopes() {
static void routes_survive_eviction_and_full_sync() {
Fixture f;
assert(sizeof(ContactInfo) <= 100); // all 350 records retain compact handles
assert(sizeof(ContactInfo) <= 92); // no per-contact padding tax across all 350 cached records
const unsigned reads = SPIFFS.reads;
f.check(0, route(0));
assert(SPIFFS.reads == reads + 1);
+3 -1
View File
@@ -101,7 +101,9 @@ class CompanionContactLifetimeTests(unittest.TestCase):
work = Path(directory)
(work / "production_constants.inc").write_text(constants, encoding="ascii")
(work / "production_types.inc").write_text(
"namespace mesh {\n" + extract_braced((ROOT / "src/Mesh.h").read_text(), "class GroupChannel")
base_header[base_header.index("#if MAX_CONTACTS + MAX_ANON_CONTACTS <= 65535"):
base_header.index("#ifndef MAX_CONNECTIONS")]
+ "namespace mesh {\n" + extract_braced((ROOT / "src/Mesh.h").read_text(), "class GroupChannel")
+ ";\n}\n" + extract_braced((ROOT / "src/helpers/ChannelDetails.h").read_text(), "struct ChannelDetails")
+ ";\n" + extract_braced(base_header, "struct ConnectionInfo") + ";\n", encoding="ascii")
(work / "production_entry.inc").write_text(
+18 -1
View File
@@ -8,6 +8,7 @@ import shutil
import subprocess
import tempfile
import unittest
from unittest.mock import patch
import zlib
from test_message_navigation import PAIRING_PREAMBLE as PREAMBLE
@@ -126,7 +127,23 @@ class CompanionReaderTest(unittest.TestCase):
packed, blocks = packer.pack(self.document)
self.assertEqual(sum(block[2] for block in blocks), 96568)
self.assertEqual(len(blocks), 24)
self.assertEqual(len(packed) + 12 * len(blocks), 38654)
self.assertEqual(len(packed), 38293)
self.assertEqual(packer.block_index_layout(blocks), ("uint16_t", 8))
self.assertEqual(len(packed) + 8 * len(blocks), 38485)
def test_index_offsets_widen_before_any_block_start_can_truncate(self):
self.assertEqual(packer.block_index_layout([(65535, 7, 2, 0)]), ("uint16_t", 8))
self.assertEqual(packer.block_index_layout([(65536, 7, 2, 0)]), ("uint32_t", 12))
self.assertEqual(packer.block_index_layout([(0xFFFFFFFF, 7, 2, 0)]), ("uint32_t", 12))
for offset in (-1, 0x100000000):
with self.assertRaises(ValueError):
packer.block_index_layout([(offset, 7, 2, 0)])
with patch.object(packer, "pack", return_value=(b"", [(65536, 7, 2, 0)])):
header = packer.render(self.document)
self.assertIn("using ReaderBlockIndex = BlockIndexT<uint32_t>;", header)
self.assertIn("using ReaderCorpus = CorpusT<uint32_t>;", header)
self.assertIn("sizeof(ReaderBlockIndex) == 12", header)
self.assertIn("{65536, 7, 2, 0}", header)
def test_invalid_sources_are_rejected(self):
for bad_text in ("", "leading\ncommand", "bad\x1b[2J", "bad\0text",
+271
View File
@@ -0,0 +1,271 @@
#!/usr/bin/env python3
"""Execute production contact indexing at compact/wide and PSRAM boundaries."""
from pathlib import Path
import os
import re
import shutil
import subprocess
import sys
import tempfile
import unittest
from test_replay_reset_integration import extract_braced
ROOT = Path(__file__).resolve().parents[1]
HARNESS = r'''
#include <cassert>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <algorithm>
#include <memory>
#include <new>
#include <set>
#include <type_traits>
#include <utility>
#include <vector>
#include <helpers/ContactListOrder.h>
#define MESH_DEBUG_PRINTLN(...) ((void)0)
@INDEX_TYPES@
struct ContactInfo {
struct Identity {
uint8_t prefix = 0;
uint32_t ordinal = 0;
bool isHashMatch(const uint8_t* hash) const { return prefix == *hash; }
} id;
uint8_t flags = 0;
uint32_t last_advert_timestamp = 0;
};
@VISITOR@
static unsigned allocations = 0, fail_allocation = 0;
static std::set<void*> outstanding;
static std::vector<std::pair<size_t, size_t>> allocation_sizes;
static constexpr int MALLOC_CAP_SPIRAM = 1, MALLOC_CAP_8BIT = 2;
static void* heap_caps_calloc(size_t count, size_t size, int) {
allocation_sizes.push_back({count, size});
if (++allocations == fail_allocation) return nullptr;
void* pointer = calloc(count, size);
assert(pointer != nullptr);
outstanding.insert(pointer);
return pointer;
}
static void heap_caps_free(void* pointer) {
assert(outstanding.erase(pointer) == 1);
free(pointer);
}
class BaseChatMesh {
public:
@STORAGE@
unsigned relocations = 0;
BaseChatMesh() {
#if defined(ESP32_PLATFORM) && defined(BOARD_HAS_PSRAM)
contacts = contacts_fallback;
sort_array = sort_array_fallback;
contact_capacity = CONTACT_PSRAM_FALLBACK_TOTAL_SLOTS;
#endif
}
~BaseChatMesh() {
#if defined(ESP32_PLATFORM) && defined(BOARD_HAS_PSRAM)
if (contacts != contacts_fallback) {
heap_caps_free(contacts);
heap_caps_free(sort_array);
}
#endif
}
static void resetContactValue(ContactInfo& contact) { contact = ContactInfo(); }
void onContactReferenceChanged(const ContactInfo*, ContactInfo*) { ++relocations; }
bool initializeContactStorage();
int searchPeersByHash(const uint8_t* hash);
void scanRecentContacts(int last_n, ContactVisitor* visitor);
};
@METHODS@
struct Visitor : ContactVisitor {
std::vector<ContactInfo> seen;
void onContactVisit(const ContactInfo& contact) override { seen.push_back(contact); }
};
static void checkSort(BaseChatMesh& mesh, int requested, size_t expected) {
Visitor visitor;
mesh.scanRecentContacts(requested, &visitor);
assert(visitor.seen.size() == expected);
std::set<uint32_t> ordinals;
for (size_t i = 0; i < visitor.seen.size(); ++i) {
const auto& contact = visitor.seen[i];
assert(contact.id.ordinal < static_cast<uint32_t>(mesh.num_contacts));
assert(ordinals.insert(contact.id.ordinal).second);
if (i > 0) {
const auto& previous = visitor.seen[i - 1];
const bool previous_favorite = (previous.flags & 1) != 0;
const bool favorite = (contact.flags & 1) != 0;
assert(previous_favorite || !favorite);
if (previous_favorite == favorite)
assert(previous.last_advert_timestamp >= contact.last_advert_timestamp);
}
}
}
static void checkFullTable() {
auto mesh = std::make_unique<BaseChatMesh>();
assert(mesh->initializeContactStorage());
mesh->num_contacts = MAX_CONTACTS + MAX_ANON_CONTACTS;
for (int i = 0; i < mesh->num_contacts; ++i) {
mesh->contacts[i].id.ordinal = i;
mesh->contacts[i].flags = (i % 7 == 0 ? 1 : 0) | 0x20;
mesh->contacts[i].last_advert_timestamp = (static_cast<uint32_t>(i) * 7919) % 997;
}
checkSort(*mesh, 0, mesh->num_contacts);
checkSort(*mesh, 11, 11);
checkSort(*mesh, mesh->num_contacts + 1, mesh->num_contacts);
// Twenty late persistent matches precede the eight transient matches. The
// 24-result limit must preserve the highest index and then four transients.
const uint8_t hash = 0x72;
for (int i = 0; i < MAX_ANON_CONTACTS; ++i) mesh->contacts[i].id.prefix = hash;
for (int i = mesh->num_contacts - 20; i < mesh->num_contacts; ++i)
mesh->contacts[i].id.prefix = hash;
assert(mesh->searchPeersByHash(&hash) == MAX_SEARCH_RESULTS);
for (int i = 0; i < 20; ++i)
assert(mesh->matching_peer_indexes[i] == mesh->num_contacts - 20 + i);
for (int i = 20; i < MAX_SEARCH_RESULTS; ++i)
assert(mesh->matching_peer_indexes[i] == i - 20);
for (int i = mesh->num_contacts - 24; i < mesh->num_contacts - 20; ++i)
mesh->contacts[i].id.prefix = hash;
assert(mesh->searchPeersByHash(&hash) == MAX_SEARCH_RESULTS);
for (int i = 0; i < MAX_SEARCH_RESULTS; ++i)
assert(mesh->matching_peer_indexes[i] == mesh->num_contacts - 24 + i);
const uint8_t missing = 0x91;
assert(mesh->searchPeersByHash(&missing) == 0);
}
#if defined(ESP32_PLATFORM) && defined(BOARD_HAS_PSRAM)
static void checkPsram() {
for (unsigned fail : {1U, 2U, 0U}) {
allocations = 0;
fail_allocation = fail;
allocation_sizes.clear();
{
BaseChatMesh mesh;
mesh.num_contacts = MAX_ANON_CONTACTS + 16;
for (int i = 0; i < mesh.num_contacts; ++i) mesh.contacts[i].id.ordinal = i + 100;
assert(mesh.initializeContactStorage() == (fail == 0));
assert(allocations == 2);
assert(allocation_sizes[1].first == MAX_CONTACTS + MAX_ANON_CONTACTS);
assert(allocation_sizes[1].second == sizeof(ContactIndex));
assert(mesh.contact_capacity == (fail ? MAX_ANON_CONTACTS + 16
: MAX_CONTACTS + MAX_ANON_CONTACTS));
assert((mesh.contacts == mesh.contacts_fallback) == (fail != 0));
assert((mesh.sort_array == mesh.sort_array_fallback) == (fail != 0));
for (int i = 0; i < mesh.num_contacts; ++i) assert(mesh.contacts[i].id.ordinal == i + 100);
if (!fail) {
const unsigned before = allocations;
assert(mesh.initializeContactStorage());
assert(allocations == before);
}
}
assert(outstanding.empty());
}
fail_allocation = 0;
}
#endif
int main() {
#if MAX_CONTACTS + MAX_ANON_CONTACTS <= 65535
static_assert(std::is_same<ContactIndex, uint16_t>::value, "compact contact indices");
#else
static_assert(std::is_same<ContactIndex, int>::value, "retain wide contact indices");
#endif
static_assert(sizeof(((BaseChatMesh*)nullptr)->matching_peer_indexes)
== MAX_SEARCH_RESULTS * sizeof(ContactIndex), "matching index storage");
#if defined(ESP32_PLATFORM) && defined(BOARD_HAS_PSRAM)
static_assert(std::is_same<decltype(BaseChatMesh::sort_array), ContactIndex*>::value,
"PSRAM sort allocation type");
static_assert(sizeof(((BaseChatMesh*)nullptr)->sort_array_fallback)
== (MAX_ANON_CONTACTS + 16) * sizeof(ContactIndex), "fallback index storage");
checkPsram();
#else
static_assert(sizeof(((BaseChatMesh*)nullptr)->sort_array)
== (MAX_CONTACTS + MAX_ANON_CONTACTS) * sizeof(ContactIndex), "sort index storage");
#endif
checkFullTable();
assert(outstanding.empty());
puts("PASS: contact sorting, peer collisions, widths and allocation ownership");
}
'''
class ContactIndexTests(unittest.TestCase):
def test_contact_layout_on_all_platforms(self):
# Compile the real record rather than the indexing harness's small
# stand-in. Only cached nRF52 records have the extra storage-slot field
# whose placement can cost four bytes per contact.
compiler = os.environ.get("CXX") or shutil.which("g++") or shutil.which("clang++")
self.assertIsNotNone(compiler, "a host C++17 compiler is required")
for platform in ("NRF52_PLATFORM", "ESP32_PLATFORM", "RP2040_PLATFORM", "STM32_PLATFORM"):
for cache in (0, 1):
expected = 92 if cache else (188 if platform == "NRF52_PLATFORM" else 184)
with self.subTest(platform=platform, cache=cache):
source = ("#include <helpers/ContactInfo.h>\n"
f'static_assert(sizeof(ContactInfo) == {expected}, "contact RAM budget");\n')
compiled = subprocess.run([
compiler, "-std=c++17", "-fsyntax-only", "-x", "c++", "-",
f"-D{platform}=1", f"-DMESH_CONTACT_CACHE={cache}",
f"-I{ROOT / 'test/fixtures/contact_cache/mocks'}",
f"-I{ROOT / 'lib/ed25519'}", f"-I{ROOT / 'src'}",
], input=source, capture_output=True, text=True, timeout=30)
self.assertEqual(compiled.returncode, 0, compiled.stdout + compiled.stderr)
def test_production_indices(self):
compiler = os.environ.get("CXX") or shutil.which("g++") or shutil.which("clang++")
self.assertIsNotNone(compiler, "a host C++17 compiler is required")
header = (ROOT / "src/helpers/BaseChatMesh.h").read_text()
source = (ROOT / "src/helpers/BaseChatMesh.cpp").read_text()
index_types = "\n".join(re.findall(
r"^#define MAX_(?:ANON_CONTACTS|SEARCH_RESULTS)\s+.+$", header, re.MULTILINE))
index_types += "\n" + header[header.index("#if MAX_CONTACTS + MAX_ANON_CONTACTS <= 65535"):
header.index("#ifndef MAX_CONNECTIONS")]
storage_start = header.index("#if defined(ESP32_PLATFORM)",
header.index("friend class ContactsIterator;"))
storage = header[storage_start:header.index(" unsigned long txt_send_timeout;", storage_start)]
methods = "\n".join(extract_braced(source, signature) for signature in (
"bool BaseChatMesh::initializeContactStorage(",
"int BaseChatMesh::searchPeersByHash(",
"static int cmp_contact_list_order(",
"void BaseChatMesh::scanRecentContacts("))
methods = "static ContactInfo* table;\n" + methods
harness = (HARNESS.replace("@INDEX_TYPES@", index_types)
.replace("@VISITOR@", extract_braced(header, "class ContactVisitor") + ";")
.replace("@STORAGE@", storage).replace("@METHODS@", methods))
variants = ((350, None, False, False),
(350, "NRF52_PLATFORM", False, True),
(350, "ESP32_PLATFORM", False, True),
(350, "ESP32_PLATFORM", True, False),
(350, "ESP32_PLATFORM", True, True),
(350, "RP2040_PLATFORM", False, False),
(32, "STM32_PLATFORM", False, False),
(65527, None, False, False), (65528, None, False, False),
(65529, None, False, False), (65529, "ESP32_PLATFORM", True, True))
with tempfile.TemporaryDirectory(prefix="meshcore-contact-indices-") as directory:
work = Path(directory)
test_source = work / "test.cpp"
test_source.write_text(harness, encoding="ascii")
for contacts, platform, psram, cache in variants:
with self.subTest(contacts=contacts, platform=platform, psram=psram, cache=cache):
binary = work / f"indices-{contacts}-{platform}-{psram}-{cache}"
command = [compiler, "-std=c++17", "-Wall", "-Wextra", "-Werror",
"-Wno-unused-function", "-Wno-sign-compare",
f"-DMAX_CONTACTS={contacts}", f"-DMESH_CONTACT_CACHE={int(cache)}",
f"-I{ROOT / 'src'}", str(test_source), "-o", str(binary)]
if platform:
command.append(f"-D{platform}=1")
if psram:
command.append("-DBOARD_HAS_PSRAM=1")
if sys.platform.startswith("linux"):
command += ["-fsanitize=address,undefined", "-fno-sanitize-recover=all",
"-fno-pie", "-no-pie"]
built = subprocess.run(command, capture_output=True, text=True, timeout=60)
self.assertEqual(built.returncode, 0, built.stdout + built.stderr)
checked = subprocess.run([str(binary)], capture_output=True, text=True, timeout=20)
self.assertEqual(checked.returncode, 0, checked.stdout + checked.stderr)
self.assertIn("PASS: contact sorting", checked.stdout)
if __name__ == "__main__":
unittest.main()
+2 -2
View File
@@ -54,13 +54,13 @@ class MorseCLITests(unittest.TestCase):
"const char* path, bool& present) { if (!fs || fs->fail_presence) return false; "
"present = fs->exists(path); return true; } }\n")
transaction = (ROOT / "src/helpers/ContactFileTransaction.h").read_text(encoding="ascii").replace(
'#include "IdentityStore.h"', '#include <helpers/IdentityStore.h>')
'#include "IdentityStore.h"', '#include <helpers/IdentityStore.h>').replace(
'#include "PersistentStoreFormat.h"', '#include <helpers/PersistentStoreFormat.h>')
headers = {
"helpers/IdentityStore.h": fs_mock,
"FilePresence.h": presence,
"AtomicFileWriter.h": (ROOT / "src/helpers/AtomicFileWriter.h").read_text(encoding="ascii"),
"ContactFileTransaction.h": transaction,
"PersistentStoreFormat.h": (ROOT / "src/helpers/PersistentStoreFormat.h").read_text(encoding="ascii"),
}
prefix = b"MC\x01\x19" + (902100).to_bytes(4, "little")
golden = prefix + (zlib.crc32(prefix) ^ 0xffffffff).to_bytes(4, "little")