Extend forwarding filters and save moderation rules atomically
This commit is contained in:
mikecarper committed
1 parent
585393e36c
commit
953ad6defb
31 files changed
+3453
-342
No files matched your search
@@ -0,0 +1,221 @@
|
||||
"""Exercise the production bounded matcher, including hostile regex-like input."""
|
||||
from pathlib import Path
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
PROGRAM = r'''
|
||||
#include <cassert>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <iostream>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include "FloodTextPattern.h"
|
||||
using namespace mesh::flood;
|
||||
static bool forbidAllocation = false;
|
||||
__attribute__((noinline)) void* operator new(size_t size) {
|
||||
assert(!forbidAllocation);
|
||||
void* result = malloc(size ? size : 1);
|
||||
if (!result) std::abort();
|
||||
return result;
|
||||
}
|
||||
__attribute__((noinline)) void* operator new[](size_t size) { return ::operator new(size); }
|
||||
__attribute__((noinline)) void operator delete(void* pointer) noexcept { free(pointer); }
|
||||
__attribute__((noinline)) void operator delete[](void* pointer) noexcept { free(pointer); }
|
||||
static_assert(sizeof(pattern_detail::State) == 4, "small NFA instructions");
|
||||
static_assert(sizeof(pattern_detail::State) * pattern_detail::MAX_STATES
|
||||
+ 2 * sizeof(pattern_detail::StateList) <= 640, "bounded matching scratch");
|
||||
static PatternMatchResult match(const char* pattern, const char* text, bool fold = false) {
|
||||
forbidAllocation = true;
|
||||
const auto result = matchTextPattern(pattern, strlen(pattern), reinterpret_cast<const uint8_t*>(text), strlen(text), fold);
|
||||
forbidAllocation = false;
|
||||
return result;
|
||||
}
|
||||
static void checks() {
|
||||
using R = PatternMatchResult;
|
||||
assert(match("", "") == R::Match);
|
||||
assert(match("Alice", "a message from Alice today") == R::Match);
|
||||
assert(match("^Alice$", "Alice") == R::Match);
|
||||
assert(match("^Alice$", "alice") == R::NoMatch);
|
||||
assert(match("^Alice$", "alice", true) == R::Match);
|
||||
assert(match("^Alice$", "Alice2") == R::NoMatch);
|
||||
assert(match("^Alice$", "2Alice") == R::NoMatch);
|
||||
assert(match("Alice$", "Alice\n") == R::NoMatch);
|
||||
assert(match("A.*e", "A\ne") == R::Match);
|
||||
assert(match("^a\\.b\\*\\[\\]\\(\\)\\|\\+\\?\\^\\$\\\\$", "a.b*[]()|+?^$\\") == R::Match);
|
||||
assert(match("^[A-F0-9]+$", "aBcF09", true) == R::Match);
|
||||
assert(match("^[A-F0-9]+$", "aBcF09") == R::NoMatch);
|
||||
assert(match("^[^A-F]+$", "xyz", true) == R::Match);
|
||||
assert(match("^[^A-F]+$", "xyzA", true) == R::NoMatch);
|
||||
assert(match("^[A-z]+$", "[_Zaz", true) == R::Match);
|
||||
assert(match("^[\\]\\-\\[]+$", "]-[") == R::Match);
|
||||
assert(match("^[a-]+$", "a--") == R::Match);
|
||||
assert(match("^[-a]+$", "a--") == R::Match);
|
||||
assert(match("^(Alice|Bob)[0-9]?$", "Bob3") == R::Match);
|
||||
assert(match("^(Alice|Bob)[0-9]?$", "Bob33") == R::NoMatch);
|
||||
assert(match("^(ab)*c+$", "ababc") == R::Match);
|
||||
assert(match("^(ab)+c?$", "") == R::NoMatch);
|
||||
assert(match("^(ab)*c?$", "") == R::Match);
|
||||
assert(match("^(a|ab)*b$", "aabab") == R::Match);
|
||||
assert(match("^(()*)*$", "") == R::Match); // epsilon cycles terminate
|
||||
assert(match("^(a|)*$", "aaaa") == R::Match);
|
||||
for (const char* invalid : {"[", "[]", "[^]", "[z-a]", "[a-\\]]", "\\", "(", ")", "a)",
|
||||
"a**", "a+?", "a??", "*a", "^*", "$+", "a]"}) {
|
||||
assert(!textPatternValid(invalid, strlen(invalid)));
|
||||
assert(match(invalid, "anything") == R::Invalid);
|
||||
}
|
||||
assert(!textPatternValid(nullptr, 0));
|
||||
assert(matchTextPattern("a", 1, nullptr, 1) == R::Invalid);
|
||||
assert(matchTextPattern("", 0, nullptr, 0) == R::Match);
|
||||
const char nulPattern[] = {'a', 0, 'b'};
|
||||
assert(!textPatternValid(nulPattern, sizeof(nulPattern)));
|
||||
const uint8_t nulText[] = {'a', 0, 'b'};
|
||||
assert(matchTextPattern("^a.b$", 5, nulText, 3) == R::Match);
|
||||
assert(matchTextPattern("^a$", 3, nulText, 3) == R::NoMatch);
|
||||
const std::string maximum(MAX_TEXT_PATTERN_LENGTH, 'a');
|
||||
assert(textPatternValid(maximum.data(), maximum.size()));
|
||||
assert(!textPatternValid(maximum.data(), maximum.size() + 1));
|
||||
assert(!textPatternValid("(((((((((a)))))))))", 19));
|
||||
assert(textPatternValid("((((((((a))))))))", 17));
|
||||
const std::string tooManyStates(MAX_TEXT_PATTERN_LENGTH, '|');
|
||||
assert(!textPatternValid(tooManyStates.data(), tooManyStates.size()));
|
||||
const std::string longSender = std::string(200, 'x') + "Alice";
|
||||
assert(match("Alice$", longSender.c_str()) == R::Match);
|
||||
assert(match("^x+$", longSender.c_str()) == R::NoMatch);
|
||||
uint32_t budget = 0;
|
||||
assert(matchTextPattern("a", 1, reinterpret_cast<const uint8_t*>("a"), 1, false, &budget) == R::BudgetExceeded);
|
||||
assert(budget == 0);
|
||||
budget = 100;
|
||||
assert(matchTextPattern("a", 1, reinterpret_cast<const uint8_t*>("a"), 1, false, &budget) == R::Match);
|
||||
assert(budget < 100 && budget > 0);
|
||||
const uint32_t afterFirst = budget;
|
||||
assert(matchTextPattern("b", 1, reinterpret_cast<const uint8_t*>("b"), 1, false, &budget) == R::Match);
|
||||
assert(budget < afterFirst);
|
||||
const std::string hostile(100000, 'a');
|
||||
budget = 500;
|
||||
assert(matchTextPattern("(a|aa)*b", 8, reinterpret_cast<const uint8_t*>(hostile.data()), hostile.size(), false, &budget) == R::BudgetExceeded);
|
||||
assert(budget == 0);
|
||||
// This pattern is exponential with a backtracking engine. The NFA finishes
|
||||
// within the default fixed budget for the largest packet-sized payload.
|
||||
const std::string packet(255, 'a');
|
||||
assert(match("^(a|aa)*b$", packet.c_str()) == R::NoMatch);
|
||||
}
|
||||
static std::vector<uint8_t> unhex(const std::string& text) {
|
||||
std::vector<uint8_t> result;
|
||||
if (text == "-") return result;
|
||||
for (size_t index = 0; index < text.size(); index += 2) {
|
||||
result.push_back(uint8_t(std::stoul(text.substr(index, 2), nullptr, 16)));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
int main() {
|
||||
checks();
|
||||
std::string patternHex, textHex;
|
||||
bool ignoreCase;
|
||||
while (std::cin >> patternHex >> textHex >> ignoreCase) {
|
||||
const auto pattern = unhex(patternHex);
|
||||
const auto text = unhex(textHex);
|
||||
// Exact-sized allocations let ASan detect an accidental C-string read.
|
||||
char* rawPattern = new char[pattern.size() ? pattern.size() : 1];
|
||||
uint8_t* rawText = new uint8_t[text.size() ? text.size() : 1];
|
||||
if (!pattern.empty()) memcpy(rawPattern, pattern.data(), pattern.size());
|
||||
if (!text.empty()) memcpy(rawText, text.data(), text.size());
|
||||
const auto result = matchTextPattern(rawPattern, pattern.size(), rawText, text.size(), ignoreCase);
|
||||
std::cout << int(result) << '\n';
|
||||
delete[] rawPattern;
|
||||
delete[] rawText;
|
||||
}
|
||||
}
|
||||
'''
|
||||
|
||||
ARM_PROGRAM = r'''
|
||||
#include "FloodTextPattern.h"
|
||||
extern "C" __attribute__((noinline)) mesh::flood::PatternMatchResult packet_match(
|
||||
const char* pattern, size_t patternLength, const uint8_t* text, size_t textLength, uint32_t* budget) {
|
||||
return mesh::flood::matchTextPattern(pattern, patternLength, text, textLength, false, budget);
|
||||
}
|
||||
'''
|
||||
|
||||
|
||||
class FloodTextPatternTests(unittest.TestCase):
|
||||
def test_packet_patterns_bounds_and_reference_agreement(self):
|
||||
rng = random.Random(436)
|
||||
patterns = ["", "a", "^a$", "a|b", "^(ab|a)*b$", "a?b+c*", "[ab]+", "[^a]*b$", "[A-z]+"]
|
||||
atoms = ["a", "b", "c", ".", "[ab]", "[^c]", "[a-c]", "[0-2]", r"\.", "(ab|c)", "(a|)"]
|
||||
for _ in range(300):
|
||||
pattern = "^" if rng.randrange(2) else ""
|
||||
for _ in range(rng.randrange(1, 5)):
|
||||
pattern += rng.choice(atoms) + rng.choice(["", "", "?", "*", "+"])
|
||||
if rng.randrange(2):
|
||||
pattern += "$"
|
||||
patterns.append(pattern)
|
||||
cases = []
|
||||
for pattern in patterns:
|
||||
for _ in range(15):
|
||||
text = "".join(rng.choice("abcABC012.") for _ in range(rng.randrange(21)))
|
||||
fold = bool(rng.randrange(2))
|
||||
expected = bool(re.search(pattern, text, re.DOTALL | (re.IGNORECASE | re.ASCII if fold else 0)))
|
||||
cases.append((pattern, text, fold, expected))
|
||||
encoded = "".join(f"{pattern.encode().hex() or '-'} {text.encode().hex() or '-'} {int(fold)}\n"
|
||||
for pattern, text, fold, _ in cases)
|
||||
# Deterministic malformed-input fuzzing exercises validation and patch
|
||||
# list bounds without assuming Python supports exactly our byte syntax.
|
||||
fuzz = ["".join(rng.choice("abc^$.*+?()[\\]|-") for _ in range(rng.randrange(1, 64)))
|
||||
for _ in range(4000)]
|
||||
encoded += "".join(f"{pattern.encode().hex()} 616263 0\n" for pattern in fuzz)
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
folder = Path(folder)
|
||||
source = folder / "patterns.cpp"
|
||||
source.write_text(PROGRAM)
|
||||
binary = folder / "patterns"
|
||||
flags = []
|
||||
if os.name != "nt":
|
||||
flags = ["-fsanitize=address,undefined", "-fno-omit-frame-pointer", "-fno-pie", "-no-pie"]
|
||||
subprocess.run(["g++", "-std=c++11", "-O1", "-Wall", "-Wextra", "-Werror", *flags,
|
||||
"-I" + str(ROOT / "src/helpers"), str(source), "-o", str(binary)], check=True)
|
||||
result = subprocess.run([str(binary)], input=encoded, text=True, capture_output=True, check=True, timeout=30)
|
||||
results = result.stdout.splitlines()
|
||||
self.assertEqual(len(results), len(cases) + len(fuzz))
|
||||
for value, case in zip(results, cases):
|
||||
self.assertEqual(int(value), int(case[3]), case)
|
||||
self.assertTrue(all(value in ("0", "1", "2") for value in results[len(cases):]))
|
||||
|
||||
def test_arm_stack_is_bounded(self):
|
||||
compiler = shutil.which("arm-none-eabi-g++")
|
||||
if not compiler:
|
||||
packages = Path.home() / ".platformio/packages"
|
||||
candidates = sorted(packages.glob("toolchain-gccarmnoneeabi*/bin/arm-none-eabi-g++*"))
|
||||
candidates = [path for path in candidates if path.suffix in ("", ".exe")]
|
||||
if candidates:
|
||||
compiler = str(candidates[0])
|
||||
if not compiler:
|
||||
self.skipTest("ARM compiler unavailable")
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
folder = Path(folder)
|
||||
source = folder / "patterns.cpp"
|
||||
source.write_text(ARM_PROGRAM)
|
||||
subprocess.run([compiler, "-std=c++11", "-mcpu=cortex-m4", "-mthumb", "-Os", "-fstack-usage",
|
||||
"-I" + str(ROOT / "src/helpers"), "-c", str(source), "-o", str(folder / "patterns.o")], check=True)
|
||||
frames = {}
|
||||
for line in (folder / "patterns.su").read_text().splitlines():
|
||||
name, size, kind = line.rsplit("\t", 2)
|
||||
self.assertEqual(kind, "static", line)
|
||||
frames[name] = int(size)
|
||||
self.assertTrue(frames)
|
||||
# Compiler recursion is limited to eight groups. Conservatively
|
||||
# count each parser frame at every level, plus runtime scratch.
|
||||
recursive = sum(size for name, size in frames.items()
|
||||
if any(token in name for token in ("::atom(", "::sequence(", "::expression(")))
|
||||
other = max(size for name, size in frames.items()
|
||||
if not any(token in name for token in ("::atom(", "::sequence(", "::expression(")))
|
||||
self.assertLessEqual(recursive * 9 + other + 64, 2048, frames)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user