Keep retry cancellation and radio recovery scoped to their owners
Cancel packets retained by the radio driver when their mesh retry is retired. Echo-confirmed delivery completes application replies; explicit cancellation releases failure references. Let active transmissions finish and prevent a cancelled packet from returning after a driver timeout. Preserve final direct-echo waits when only the other radio profile or crossover policy changes. Recover verified radio-profile settings from backup even when filesystem repair fails, and protect the backup after a failed save rollback. Add regressions for direct and flood cancellation, confirmed command replies, active TX completion and timeouts, subsequent packet cleanup, independent profile echo ownership, and storage rename/removal failures.
This commit is contained in:
1 parent
dde87f1d15
commit
e720132a8b
8 files changed
+274
-29
No files matched your search
+45
@@ -43,6 +43,51 @@ struct Fixture {
|
||||
void advance(uint32_t ms, bool epoch=true) { g_mock_millis+=ms; if(epoch)clock.epoch+=ms/1000; cli.loop(); }
|
||||
};
|
||||
int main() {
|
||||
for (bool damaged_primary : {false, true}) {
|
||||
Fixture f(true);
|
||||
f.cmd("set radio2 910.5,500,8,5,rxtx,80");
|
||||
f.cmd("set radio2.cross off");
|
||||
f.cmd("set tx.reply radio2 force");
|
||||
assert(f.cli.savePrimaryPreamble(120));
|
||||
assert(f.fs.rename("/radio_profiles", "/radio_profiles.bak"));
|
||||
const auto backup = f.fs.files["/radio_profiles.bak"];
|
||||
if (damaged_primary) {
|
||||
f.fs.files["/radio_profiles"] = {0};
|
||||
f.fs.fail_remove = true;
|
||||
} else {
|
||||
f.fs.fail_rename = 1;
|
||||
}
|
||||
Radio radio; mesh::RadioProfileCLI restored;
|
||||
restored.begin(&f.fs, &radio, &f.clock, true);
|
||||
// The verified backup stays usable even when storage cannot repair it.
|
||||
assert(radio.p.enabled());
|
||||
assert(radio.p.secondary.params.freq == 910.5f);
|
||||
assert(radio.p.secondary.params.preamble == 80);
|
||||
assert(restored.primaryPreamble() == 120);
|
||||
assert(radio.p.cross == mesh::RadioCrossMode::Off);
|
||||
assert(radio.p.reply_tx == mesh::RADIO_TX_SECONDARY && radio.p.reply_force);
|
||||
f.fs.fail_remove = false;
|
||||
assert(restored.handle("set radio2 off", f.reply));
|
||||
assert(strstr(f.reply, "Error"));
|
||||
assert(f.fs.files["/radio_profiles.bak"] == backup);
|
||||
}
|
||||
{
|
||||
Fixture f;
|
||||
f.cmd("set radio2 910.5,500,8,5,rxtx,80"); f.advance(2000);
|
||||
const auto committed = f.fs.files["/radio_profiles"];
|
||||
f.fs.fail_rename_from = {"/radio_profiles.tmp", "/radio_profiles.bak"};
|
||||
f.cmd("set radio2 off", false); // both publication and rollback fail
|
||||
assert(f.radio.p.enabled());
|
||||
assert(f.fs.files["/radio_profiles.bak"] == committed);
|
||||
f.fs.fail_rename_from.clear();
|
||||
f.cmd("set radio2.cross on", false); // protect the sole committed copy
|
||||
assert(f.fs.files["/radio_profiles.bak"] == committed);
|
||||
Radio radio; mesh::RadioProfileCLI restored;
|
||||
restored.begin(&f.fs, &radio, &f.clock);
|
||||
assert(radio.p.enabled() && radio.p.secondary.params.preamble == 80);
|
||||
assert(restored.handle("set radio2 off", f.reply));
|
||||
assert(!strncmp(f.reply, "OK", 2)); // reboot repairs storage and releases the hold
|
||||
}
|
||||
{
|
||||
Fixture f(true);
|
||||
f.cmd("set tx.reply off");
|
||||
|
||||
+176
@@ -13,6 +13,8 @@ class DualProfileTestRadio : public RetryCodingRateRadio {
|
||||
void loop() override { ++carrier_services; }
|
||||
bool recoverRadio(bool) override { ++recoveries; return true; }
|
||||
uint32_t estimated_airtime = 10;
|
||||
float receive_score = 0;
|
||||
float packetScore(float, int) override { return receive_score; }
|
||||
TraceTestClock* sensing_clock = nullptr;
|
||||
uint32_t sensing_delay = 0;
|
||||
bool distinguish_airtime = false;
|
||||
@@ -72,9 +74,26 @@ class DualProfileTestMesh : public RetryCodingRateMesh {
|
||||
using RetryCodingRateMesh::getRetransmitDelay;
|
||||
using RetryCodingRateMesh::getOtaPacketAirtime;
|
||||
using RetryCodingRateMesh::tryParsePacket;
|
||||
using RetryCodingRateMesh::cancelAllDirectRetries;
|
||||
using RetryCodingRateMesh::cancelAllFloodRetries;
|
||||
bool cross_filter_allows = true;
|
||||
float ota_speed = 1.0f;
|
||||
bool suppress_tx = false;
|
||||
unsigned send_failures = 0;
|
||||
unsigned send_completions = 0;
|
||||
void onSendComplete(mesh::Packet* packet) override {
|
||||
++send_completions;
|
||||
RetryCodingRateMesh::onSendComplete(packet);
|
||||
}
|
||||
unsigned tx_failure_logs = 0;
|
||||
void logTxFail(mesh::Packet*, int) override { ++tx_failure_logs; }
|
||||
unsigned direct_successes = 0, direct_failures = 0;
|
||||
void onDirectRetrySucceeded(const uint8_t*, uint8_t, int8_t) override { ++direct_successes; }
|
||||
void onDirectRetryFailed(const uint8_t*, uint8_t) override { ++direct_failures; }
|
||||
void onSendFail(mesh::Packet* packet) override {
|
||||
++send_failures;
|
||||
RetryCodingRateMesh::onSendFail(packet);
|
||||
}
|
||||
bool allowPacketTransmit(const mesh::Packet* packet) const override {
|
||||
return !suppress_tx && RetryCodingRateMesh::allowPacketTransmit(packet);
|
||||
}
|
||||
@@ -249,6 +268,121 @@ TEST_F(DualProfileTest, RadioFaultRetryRetainsBoundedBusyEscapeAndSingleRetryLim
|
||||
EXPECT_TRUE(radio.transmissions.empty());
|
||||
}
|
||||
|
||||
TEST_F(DualProfileTest, EchoDuringRadioFaultBackoffCancelsDirectAndFloodRetry) {
|
||||
radio.config.cross = mesh::RadioCrossMode::Off;
|
||||
radio.receive_score = 1; // strong echo is processed without a score delay
|
||||
for (bool direct : {false, true}) {
|
||||
const uint8_t route[] = {0x11, 0x22};
|
||||
ASSERT_NE(nullptr, direct ? makeDirectText(node, 0x55, route, sizeof(route)) : queue());
|
||||
tick(); ASSERT_TRUE(radio.sending);
|
||||
radio.complete = true; tick();
|
||||
ASSERT_EQ(manager.getOutboundTotal(), 1);
|
||||
auto* retry = manager.getOutboundByIdx(0);
|
||||
mesh::Packet echo = *retry;
|
||||
echo.setPathHashCount(retry->getPathHashCount() + (direct ? -1 : 1));
|
||||
radio.fail_next_send = true;
|
||||
tick(10000); ASSERT_TRUE(node.hasOutbound());
|
||||
ASSERT_FALSE(radio.sending);
|
||||
const auto sent = radio.transmissions.size();
|
||||
// Deliver the real downstream echo through Dispatcher::checkRecv while
|
||||
// the acknowledged retry is retained outside the outbound queue.
|
||||
radio.incoming = {echo.header, echo.path_len};
|
||||
radio.incoming.insert(radio.incoming.end(), echo.path, echo.path + echo.getPathByteLen());
|
||||
radio.incoming.insert(radio.incoming.end(), echo.payload, echo.payload + echo.payload_len);
|
||||
tick(500);
|
||||
EXPECT_TRUE(radio.incoming.empty());
|
||||
EXPECT_EQ(radio.transmissions.size(), sent);
|
||||
EXPECT_FALSE(node.hasOutbound());
|
||||
EXPECT_EQ(manager.getFreeCount(), 40);
|
||||
EXPECT_EQ(node.tx_failure_logs, 0u); // cancellation is not a TX failure
|
||||
// Keep failures independent so both packet types run in the regression.
|
||||
radio.complete = true; tick();
|
||||
node.cancelAllDirectRetries(); node.cancelAllFloodRetries();
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(DualProfileTest, ExplicitRetryCancellationRetiresRetainedPacketEvenWhenRadioBusy) {
|
||||
radio.config.cross = mesh::RadioCrossMode::Off;
|
||||
for (bool direct : {false, true}) {
|
||||
const uint8_t route[] = {0x11, 0x22};
|
||||
ASSERT_NE(nullptr, direct ? makeDirectText(node, 0x55, route, sizeof(route)) : queue());
|
||||
tick(); ASSERT_TRUE(radio.sending);
|
||||
radio.complete = true; tick();
|
||||
ASSERT_EQ(manager.getOutboundTotal(), 1);
|
||||
radio.fail_next_send = true;
|
||||
tick(10000); ASSERT_TRUE(node.hasOutbound());
|
||||
const auto sent = radio.transmissions.size();
|
||||
if (direct) node.cancelAllDirectRetries();
|
||||
else node.cancelAllFloodRetries();
|
||||
radio.hold_prepare_busy = true;
|
||||
tick(500);
|
||||
EXPECT_FALSE(node.hasOutbound());
|
||||
EXPECT_EQ(manager.getFreeCount(), 40);
|
||||
radio.hold_prepare_busy = false;
|
||||
tick(500);
|
||||
EXPECT_EQ(radio.transmissions.size(), sent);
|
||||
radio.complete = true; tick();
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(DualProfileTest, EchoCancelsRadioRecoveryOfInitialTransmission) {
|
||||
radio.config.cross = mesh::RadioCrossMode::Off;
|
||||
radio.receive_score = 1;
|
||||
for (bool direct : {false, true}) {
|
||||
const uint8_t route[] = {0x11, 0x22};
|
||||
auto* original = direct ? makeDirectText(node, 0x55, route, sizeof(route)) : queue();
|
||||
ASSERT_NE(nullptr, original);
|
||||
mesh::Packet echo = *original;
|
||||
echo.setPathHashCount(original->getPathHashCount() + (direct ? -1 : 1));
|
||||
radio.fail_next_send = true;
|
||||
tick(); ASSERT_TRUE(node.hasOutbound());
|
||||
const auto completions = node.send_completions;
|
||||
radio.incoming = {echo.header, echo.path_len};
|
||||
radio.incoming.insert(radio.incoming.end(), echo.path, echo.path + echo.getPathByteLen());
|
||||
radio.incoming.insert(radio.incoming.end(), echo.payload, echo.payload + echo.payload_len);
|
||||
tick(500);
|
||||
EXPECT_TRUE(radio.transmissions.empty());
|
||||
EXPECT_FALSE(node.hasOutbound());
|
||||
EXPECT_EQ(manager.getFreeCount(), 40);
|
||||
EXPECT_EQ(node.tx_failure_logs, 0u);
|
||||
EXPECT_EQ(node.send_failures, 0u);
|
||||
// A downstream echo proves the command reply arrived. Complete its
|
||||
// application barrier rather than cancelling the pending radio handoff.
|
||||
EXPECT_EQ(node.send_completions, completions + 1);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(DualProfileTest, RetryCancellationLetsActiveTransmitFinishButPreventsTimeoutRetry) {
|
||||
radio.config.cross = mesh::RadioCrossMode::Off;
|
||||
for (bool direct : {false, true}) for (bool timeout : {false, true}) {
|
||||
SCOPED_TRACE(testing::Message() << "direct=" << direct << " timeout=" << timeout);
|
||||
const uint8_t route[] = {0x11, 0x22};
|
||||
ASSERT_NE(nullptr, direct ? makeDirectText(node, 0x55, route, sizeof(route)) : queue());
|
||||
tick(); ASSERT_TRUE(radio.sending);
|
||||
radio.complete = true; tick();
|
||||
tick(10000); ASSERT_TRUE(radio.sending); // retry is already on air
|
||||
if (direct) node.cancelAllDirectRetries();
|
||||
else node.cancelAllFloodRetries();
|
||||
const auto sent = radio.transmissions.size();
|
||||
const auto failed = node.send_failures;
|
||||
tick(); EXPECT_TRUE(radio.sending); // do not cut an active TX short
|
||||
radio.complete = !timeout;
|
||||
tick(timeout ? 10000 : 1);
|
||||
EXPECT_FALSE(node.hasOutbound());
|
||||
EXPECT_EQ(manager.getFreeCount(), 40);
|
||||
tick(500);
|
||||
EXPECT_EQ(radio.transmissions.size(), sent);
|
||||
EXPECT_EQ(node.send_failures, failed + (timeout ? 1 : 0));
|
||||
// Cancellation must not suppress failure cleanup for the next stale
|
||||
// packet, which is rejected before the ordinary TX initialization path.
|
||||
auto* stale = queue(); ASSERT_NE(nullptr, stale);
|
||||
++stale->radio_generation;
|
||||
tick();
|
||||
EXPECT_EQ(node.send_failures, failed + (timeout ? 1 : 0) + 1);
|
||||
EXPECT_EQ(manager.getFreeCount(), 40);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(DualProfileTest, PacketBecomingReadyDuringCadCannotSkipItsOwnChannelCheck) {
|
||||
node.flood_attempts = 0;
|
||||
node.tempRadioActive = true;
|
||||
@@ -812,6 +946,48 @@ TEST_F(DualProfileTest, DirectRetriesKeepSeparateCodingRatesAndEchoOwnership) {
|
||||
EXPECT_EQ(b, manager.getOutboundByIdx(0));
|
||||
}
|
||||
|
||||
TEST_F(DualProfileTest, FinalDirectEchoWaitOnlyExpiresWhenItsOwnProfileChanges) {
|
||||
node.direct_attempts = 1;
|
||||
for (uint8_t origin : {0, 1}) for (unsigned change = 0; change < 3; ++change) {
|
||||
SCOPED_TRACE(testing::Message() << "origin=" << unsigned(origin) << " change=" << change);
|
||||
radio.config.cross = mesh::RadioCrossMode::Off;
|
||||
tick();
|
||||
auto* packet = node.obtainNewPacket(); ASSERT_NE(nullptr, packet);
|
||||
packet->header = PAYLOAD_TYPE_TXT_MSG << PH_TYPE_SHIFT;
|
||||
packet->payload_len = 3;
|
||||
packet->payload[0] = 0xa1; packet->payload[1] = 0xb2; packet->payload[2] = 0x55;
|
||||
packet->tx_radio = origin ? mesh::RADIO_TX_SECONDARY : mesh::RADIO_TX_PRIMARY;
|
||||
const uint8_t route[] = {0x11, 0x22};
|
||||
ASSERT_TRUE(node.sendDirect(packet, route, sizeof(route)));
|
||||
tick(); ASSERT_TRUE(radio.sending);
|
||||
radio.complete = true; tick();
|
||||
ASSERT_EQ(manager.getOutboundTotal(), 1);
|
||||
mesh::Packet echo = *manager.getOutboundByIdx(0);
|
||||
echo.radio_bound = false; echo.radio_local = false;
|
||||
echo.setPathHashCount(echo.getPathHashCount() - 1);
|
||||
tick(10000); ASSERT_TRUE(radio.sending);
|
||||
radio.complete = true; tick();
|
||||
ASSERT_EQ(manager.getOutboundTotal(), 0);
|
||||
ASSERT_EQ(manager.getFreeCount(), 40); // final echo waits own metadata only
|
||||
|
||||
const auto successes = node.direct_successes;
|
||||
if (change == 1) {
|
||||
radio.config.cross = mesh::RadioCrossMode::On;
|
||||
} else if ((change == 2 ? origin : origin ^ 1) == 0) {
|
||||
auto primary = radio.config.primary; primary.freq += 1;
|
||||
radio.config.setPrimary(primary, false);
|
||||
} else {
|
||||
auto secondary = radio.config.secondary; secondary.params.freq += 1;
|
||||
radio.config.setSecondary(secondary, false);
|
||||
}
|
||||
tick();
|
||||
node.receivePacket(&echo);
|
||||
EXPECT_EQ(node.direct_successes, successes + (change == 2 ? 0 : 1));
|
||||
EXPECT_EQ(node.direct_failures, 0u); // profile changes are not link failures
|
||||
EXPECT_EQ(manager.getFreeCount(), 40);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(DualProfileTest, ExpiredSecondaryPacketsAreDiscarded) {
|
||||
node.flood_attempts = 0;
|
||||
radio.config.secondary_temporary = true;
|
||||
|
||||
@@ -27,12 +27,15 @@ class MemoryFS {
|
||||
public:
|
||||
std::map<std::string, std::vector<uint8_t>> files;
|
||||
bool fail_write = false;
|
||||
bool fail_remove = false;
|
||||
int fail_rename = 0;
|
||||
std::vector<std::string> fail_rename_from;
|
||||
bool mkdir(const char*) { return true; }
|
||||
bool exists(const char* path) const { return files.count(path); }
|
||||
bool remove(const char* path) { return files.erase(path); }
|
||||
bool remove(const char* path) { return !fail_remove && files.erase(path); }
|
||||
bool rename(const char* from, const char* to) {
|
||||
if (fail_rename > 0 && --fail_rename == 0) return false;
|
||||
if (std::find(fail_rename_from.begin(), fail_rename_from.end(), from) != fail_rename_from.end()) return false;
|
||||
if (!exists(from) || exists(to)) return false;
|
||||
files[to] = files[from]; files.erase(from); return true;
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user