Isolate RAK help packing from the firmware C++11 build

.github/workflows/run-unit-tests.yml

@@ -831,6 +831,7 @@ jobs:
           python3 -B test/test_companion_board_power_commands.py -v
           python3 -B test/test_companion_radio_diagnostics.py -v
           python3 -B test/test_companion_help_writer.py -v
+          python3 -B test/test_companion_help_build_policy.py -v
           python3 -B test/test_sdcard_cli_parity.py -v
           python3 -B test/test_esp32_buffered_radio_targets.py -v
           python3 -B test/test_xiao_battery_safety.py -v

examples/companion_radio/CompanionHelpPacked.cpp

@@ -0,0 +1,32 @@
+#if defined(RAK_4631) && defined(COMPANION_RADIO_FULL)
+// This is the only Companion translation unit compiled as C++17. Include the
+// same feature/board context as MyMesh before expanding the single help source.
+#include "MyMesh.h"
+#include "CompanionHelpWriter.h"
+
+namespace mesh { namespace companion {
+size_t writePackedCompanionHelp(Stream& output) {
+    // Constant evaluation retains only the exact-sized packed help image.
+    // The source fragments and every feature gate remain shared below.
+    static constexpr auto packed_help = []() constexpr {
+      constexpr char help_text[] =
+#define MESH_COMPANION_HELP_BEGIN
+#define MESH_COMPANION_HELP_END
+#define MESH_COMPANION_HELP_LINE_BEGIN
+#define MESH_COMPANION_HELP_LINE_END "\r\n"
+#include "CompanionHelp.inc"
+#undef MESH_COMPANION_HELP_BEGIN
+#undef MESH_COMPANION_HELP_END
+#undef MESH_COMPANION_HELP_LINE_BEGIN
+#undef MESH_COMPANION_HELP_LINE_END
+          ;
+      static_assert(mesh::companion::validHelpText(help_text),
+                    "Packed help requires ASCII without embedded NUL bytes");
+      return mesh::companion::packHelp<
+          mesh::companion::packedHelpSize(help_text)>(help_text);
+    }();
+    return mesh::companion::writePackedHelp(output, packed_help.bytes,
+                                    sizeof(packed_help.bytes));
+}
+} } // namespace mesh::companion
+#endif

examples/companion_radio/CompanionHelpWriter.h

@@ -3,8 +3,13 @@
 #include <stddef.h>
 #include <stdint.h>
 
+class Stream;
+
 namespace mesh { namespace companion {
 
+// The implementation is isolated in the RAK Full C++17 packing source.
+size_t writePackedCompanionHelp(Stream& output);
+
 // Native USB accepts bounded producer writes. Honor partial progress, and stop
 // if the sink cannot accept another byte rather than spinning on a closed host.
 template<typename Output>
@@ -22,6 +27,7 @@ size_t writeHelpChunks(Output& output, const char* text, size_t length) {
   return written;
 }
 
+#if __cplusplus >= 201703L
 namespace help_packing {
 
 // Help is ASCII. Bytes 0x80..0x8f encode these common fragments; the original
@@ -135,4 +141,6 @@ size_t writePackedHelp(Output& output, const uint8_t* packed, size_t length) {
   return written;
 }
 
+#endif // C++17 constant packing and streaming decoder
+
 } } // namespace mesh::companion

examples/companion_radio/MyMesh.cpp

@@ -10405,27 +10405,7 @@ void MyMesh::handleTerminalCommand(char* command, bool reader_prompt,
   } else if (strcmp(command, "help") == 0) {
     auto& output = terminalOutput();
 #if defined(RAK_4631) && defined(COMPANION_RADIO_FULL)
-    // Constant evaluation retains only the exact-sized packed help image.
-    // The source fragments and every feature gate remain shared below.
-    static constexpr auto packed_help = []() constexpr {
-      constexpr char help_text[] =
-#define MESH_COMPANION_HELP_BEGIN
-#define MESH_COMPANION_HELP_END
-#define MESH_COMPANION_HELP_LINE_BEGIN
-#define MESH_COMPANION_HELP_LINE_END "\r\n"
-#include "CompanionHelp.inc"
-#undef MESH_COMPANION_HELP_BEGIN
-#undef MESH_COMPANION_HELP_END
-#undef MESH_COMPANION_HELP_LINE_BEGIN
-#undef MESH_COMPANION_HELP_LINE_END
-          ;
-      static_assert(mesh::companion::validHelpText(help_text),
-                    "Packed help requires ASCII without embedded NUL bytes");
-      return mesh::companion::packHelp<
-          mesh::companion::packedHelpSize(help_text)>(help_text);
-    }();
-    mesh::companion::writePackedHelp(output, packed_help.bytes,
-                                    sizeof(packed_help.bytes));
+    mesh::companion::writePackedCompanionHelp(output);
 #else
 #define MESH_COMPANION_HELP_BEGIN output.print(
 #define MESH_COMPANION_HELP_END );

scripts/nrf52_flash_trim.py

@@ -183,11 +183,43 @@ def trim_tft_library(build_env, node):
     return node
 
 
+def packed_help_policy(platform, mcu, src_filter, flags=(), cppdefines=()):
+    if platform != "nordicnrf52" or str(mcu).lower() != "nrf52840":
+        return False
+    if not source_selected(src_filter, COMPANION_SOURCES[0]):
+        return False
+    if any(source_selected(src_filter, source) for source in ROLE_SOURCES):
+        return False
+    values = flag_defines(flags, cppdefines)
+    # Match the C++ #if defined gate, including an explicit =0 definition.
+    return all(values.get(name) is not None for name in
+               ("RAK_4631", "COMPANION_RADIO_FULL"))
+
+
+def compile_packed_help(build_env, node):
+    source = Path(node.srcnode().get_abspath()).resolve()
+    expected = Path(build_env.subst("$PROJECT_DIR/examples/companion_radio/CompanionHelpPacked.cpp")).resolve()
+    if source != expected:
+        return node
+    private_env = build_env.Clone()
+    # The framework appends gnu++11 after pre-scripts. Override only this
+    # source's final environment, leaving MyMesh, libraries and all other
+    # targets on their original language standard and feature context.
+    flags = {name: [value for value in private_env.get(name, ())
+                    if not str(value).startswith("-std=")]
+             for name in ("CCFLAGS", "CXXFLAGS")}
+    flags["CXXFLAGS"].append("-std=gnu++17")
+    private_env.Replace(**flags)
+    return private_env.Object(node)
+
+
 def install(build_env):
     platform = build_env.PioPlatform()
     mcu = build_env.BoardConfig().get("build.mcu", "")
     flags = build_env.get("BUILD_FLAGS", ())
     cppdefines = build_env.get("CPPDEFINES", ())
+    if packed_help_policy(platform.name, mcu, build_env.get("SRC_FILTER", ()), flags, cppdefines):
+        build_env.AddBuildMiddleware(compile_packed_help, "*CompanionHelpPacked.cpp")
     definitions = trim_policy(platform.name, mcu, build_env["PIOENV"],
                               build_env.get("SRC_FILTER", ()), flags, cppdefines)
     infrastructure = bool(definitions)

test/test_companion_help_build_policy.py

@@ -0,0 +1,41 @@
+"""The C++17 help encoder is one RAK Full source, never a global ABI change."""
+from pathlib import Path
+import copy
+import importlib.util
+import unittest
+ROOT=Path(__file__).resolve().parents[1]
+spec=importlib.util.spec_from_file_location('help_language_policy',ROOT/'scripts/nrf52_flash_trim.py');m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m)
+class HelpBuildPolicy(unittest.TestCase):
+ def test_only_actual_rak_full_companion_profile_selects_cpp17_helper(self):
+  yes=dict(platform='nordicnrf52',mcu='nrf52840',src_filter='+<*> +<../examples/companion_radio>',flags='-DRAK_4631 -DCOMPANION_RADIO_FULL')
+  self.assertTrue(m.packed_help_policy(**yes))
+  for key,value in [('platform','espressif32'),('mcu','nrf52832'),('src_filter','+<*>'),('src_filter','+<../examples/companion_radio> +<../examples/simple_repeater>'),('flags','-DRAK_4631'),('flags','-DCOMPANION_RADIO_FULL'),('flags','-DRAK_4631 -DCOMPANION_RADIO_FULL -UCOMPANION_RADIO_FULL')]:
+   with self.subTest(key=key,value=value):
+    no=dict(yes);no[key]=value;self.assertFalse(m.packed_help_policy(**no))
+  self.assertTrue(m.packed_help_policy('nordicnrf52','nrf52840','+<../examples/companion_radio>',cppdefines=[('RAK_4631',1),('COMPANION_RADIO_FULL',1)]))
+ def test_final_object_clone_changes_only_that_translation_units_language(self):
+  class Node:
+   def __init__(self,path):self.path=path
+   def srcnode(self):return self
+   def get_abspath(self):return str(self.path)
+  class Env(dict):
+   def subst(self,text):return text.replace('$PROJECT_DIR',str(ROOT))
+   def Clone(self):self.clone=Env(copy.deepcopy(dict(self)));return self.clone
+   def Replace(self,**values):self.update(values)
+   def Object(self,node):self.object=node;return ('compiled17',node)
+  env=Env(CXXFLAGS=['-fno-exceptions','-std=gnu++11','-fno-rtti'],CCFLAGS=['-Os','-flto','-std=gnu++11'],CPPDEFINES=[('FEATURE',1)])
+  before=copy.deepcopy(dict(env));other=Node(ROOT/'examples/companion_radio/MyMesh.cpp')
+  self.assertIs(m.compile_packed_help(env,other),other);self.assertFalse(hasattr(env,'clone'))
+  node=Node(ROOT/'examples/companion_radio/CompanionHelpPacked.cpp');result=m.compile_packed_help(env,node)
+  self.assertEqual(result,('compiled17',node));self.assertEqual(dict(env),before)
+  self.assertEqual(env.clone['CXXFLAGS'],['-fno-exceptions','-fno-rtti','-std=gnu++17'])
+  self.assertEqual(env.clone['CCFLAGS'],['-Os','-flto']);self.assertEqual(env.clone['CPPDEFINES'],before['CPPDEFINES'])
+ def test_producer_uses_same_feature_context_and_single_help_source(self):
+  cpp=(ROOT/'examples/companion_radio/CompanionHelpPacked.cpp').read_text();mesh=(ROOT/'examples/companion_radio/MyMesh.cpp').read_text()
+  self.assertLess(cpp.index('#include "MyMesh.h"'),cpp.index('#include "CompanionHelp.inc"'))
+  self.assertIn('#if defined(RAK_4631) && defined(COMPANION_RADIO_FULL)',cpp)
+  self.assertEqual(cpp.count('#include "CompanionHelp.inc"'),1)
+  branch=mesh[mesh.index('    auto& output = terminalOutput();',mesh.index('} else if (strcmp(command, "help") == 0)')):]
+  self.assertIn('mesh::companion::writePackedCompanionHelp(output);',branch)
+  self.assertNotIn('constexpr',branch[:branch.index('#else')])
+if __name__=='__main__':unittest.main()

test/test_companion_help_writer.py

@@ -9,6 +9,7 @@ import subprocess
 import sys
 import tempfile
 import unittest
+from cpp_source import body
 
 ROOT = Path(__file__).resolve().parents[1]
 FEATURES = ['COMPANION_FEATURE_READER=1', 'COMPANION_FEATURE_TEMP_RADIO=1',
@@ -43,7 +44,8 @@ HARNESS = r'''
 #include <string>
 #include <vector>
 #include <utility>
-struct Output {
+class Stream {
+public:
   std::string text, trace;
   std::vector<size_t> writes;
   size_t throttle = 256, credit = size_t(-1);
@@ -67,8 +69,11 @@ struct Output {
     text.append(reinterpret_cast<const char*>(value),n);credit-=n;
     record('W',reinterpret_cast<const char*>(value),n);return n;
   }
-} output;
+};
+using Output=Stream;
+Output output;
 Output& terminalOutput() {return output;}
+@PACKED_IMPLEMENTATION@
 void actualHelp(bool _terminal_mode) {
 @ACTUAL_HELP@
 }
@@ -143,7 +148,9 @@ class CompanionHelpWriterTests(unittest.TestCase):
         start = source.index('    auto& output = terminalOutput();',
                              source.index('} else if (strcmp(command, "help") == 0)'))
         end = source.index('  } else if (handleCommand(command, 0, local_reply))', start)
-        harness = HARNESS.replace('@ACTUAL_HELP@', source[start:end])
+        implementation = body((ROOT / 'examples/companion_radio/CompanionHelpPacked.cpp').read_text(), 'size_t writePackedCompanionHelp(Stream& output)')
+        implementation = '#if defined(RAK_4631) && defined(COMPANION_RADIO_FULL)\nnamespace mesh { namespace companion {\n' + implementation + '\n} }\n#endif\n'
+        harness = HARNESS.replace('@ACTUAL_HELP@', source[start:end]).replace('@PACKED_IMPLEMENTATION@', implementation)
         for name, defines in PROFILES.items():
             with self.subTest(profile=name), tempfile.TemporaryDirectory(prefix='companion-help-') as folder:
                 work = Path(folder);(work / 'test.cpp').write_text(harness)
@@ -164,5 +171,30 @@ class CompanionHelpWriterTests(unittest.TestCase):
                     self.assertNotIn(b'Commands:\r\n  stats-core / stats-radio / stats-radio-diag',image)
 
 
+    def test_cpp11_caller_links_actual_cpp17_packing_translation_unit(self):
+        source=(ROOT/'examples/companion_radio/MyMesh.cpp').read_text()
+        start=source.index('    auto& output = terminalOutput();',source.index('} else if (strcmp(command, "help") == 0)'))
+        end=source.index('  } else if (handleCommand(command, 0, local_reply))',start)
+        consumer=HARNESS.replace('@ACTUAL_HELP@',source[start:end]).replace('@PACKED_IMPLEMENTATION@','')
+        consumer=consumer[:consumer.index('  constexpr char literal[] =')]+"}\n"
+        stream=HARNESS[HARNESS.index('class Stream {'):HARNESS.index('using Output=Stream;')]
+        headers='#include <cstdint>\n#include <algorithm>\n#include <cassert>\n#include <cstdio>\n#include <cstring>\n#include <string>\n#include <vector>\n#include <utility>\n'
+        with tempfile.TemporaryDirectory(prefix='companion-help-language-') as folder:
+            work=Path(folder)
+            (work/'MyMesh.h').write_text(headers+stream)
+            (work/'CompanionHelpPacked.cpp').write_bytes((ROOT/'examples/companion_radio/CompanionHelpPacked.cpp').read_bytes())
+            (work/'consumer.cpp').write_text(consumer)
+            compiler=os.environ.get('CXX','g++')
+            options=['-Os','-Wall','-Wextra','-Werror','-fsanitize=address,undefined','-fno-sanitize-recover=all','-fno-pie','-no-pie',*['-D'+v for v in PROFILES['rak_full']],'-I'+str(work),'-I'+str(ROOT/'examples/companion_radio')]
+            for name,standard in [('CompanionHelpPacked','c++17'),('consumer','c++11')]:
+                result=subprocess.run([compiler,'-std='+standard,*options,'-c',str(work/(name+'.cpp')),'-o',str(work/(name+'.o'))],capture_output=True,text=True)
+                self.assertEqual(result.returncode,0,result.stdout+result.stderr)
+            result=subprocess.run([compiler,*options,str(work/'consumer.o'),str(work/'CompanionHelpPacked.o'),'-o',str(work/'test')],capture_output=True,text=True)
+            self.assertEqual(result.returncode,0,result.stdout+result.stderr)
+            result=subprocess.run([str(work/'test')],capture_output=True)
+            self.assertEqual(result.returncode,0,result.stderr.decode(errors='replace'))
+            self.assertEqual(hashlib.sha256(result.stdout).hexdigest(),GOLDEN['rak_full']['text'])
+            self.assertNotIn(b'Commands:\r\n  stats-core / stats-radio / stats-radio-diag',(work/'test').read_bytes())
+
 if __name__ == '__main__':
     unittest.main()