diff --git a/.gitattributes b/.gitattributes index 575ba1c7..094d6379 100644 --- a/.gitattributes +++ b/.gitattributes @@ -14,3 +14,6 @@ tools/hil/preamble_stationary*.json -text whitespace=cr-at-eol tools/hil/sf6_125*.json -text whitespace=cr-at-eol tools/hil/sf8_125*.json -text whitespace=cr-at-eol tools/hil/sf5_250*.json -text whitespace=cr-at-eol + +# Preserve the pinned upstream FS/newlib fixtures byte-for-byte. +test/fixtures/esp32_file_buffer/* -text whitespace=-blank-at-eol,-blank-at-eof diff --git a/platformio.ini b/platformio.ini index 62afcb7a..f2cc0d8a 100644 --- a/platformio.ini +++ b/platformio.ini @@ -89,6 +89,7 @@ extra_scripts = pre:scripts/portable_esp32_link.py pre:scripts/esp32_ota_heap_context.py pre:scripts/esp32_usb_session_fix.py + pre:scripts/esp32_file_buffer_fix.py merge-bin.py post:tools/mota/pio_endf.py post:scripts/check_esp32_dram.py diff --git a/scripts/esp32_file_buffer_fix.py b/scripts/esp32_file_buffer_fix.py new file mode 100644 index 00000000..28047156 --- /dev/null +++ b/scripts/esp32_file_buffer_fix.py @@ -0,0 +1,109 @@ +"""Reject unhonored stdio buffer sizes in the pinned Arduino-ESP32 FS copy. + +Newlib can return success after falling back to a larger allocation. Keep the +shared framework intact and make File::setBufferSize fail before caller I/O. +""" +import hashlib +from pathlib import Path +import re + + +PINNED_VFS_SHA256 = "db3eb3fef8e59c0bafd01416ea18e2d5f36f38a9b180cd415ff3563a59fcfb65" +PATCHED_VFS_SHA256 = "e6d5a27af0fceb3ee937e95c42eb8341c0c0c762e4cbf9cf54fd522c8eba8d60" +SOURCE_SUFFIX = ("libraries", "FS", "src", "vfs_api.cpp") +INCLUDE = '#include "vfs_api.h"\n' +LAYOUT_GUARD = ''' +// MeshCore strict buffer admission for pinned Arduino 2.0.17/newlib 3.3.0. +// This check deliberately fails compilation if the reviewed FILE ABI changes. +#include +#include +#include +#if __NEWLIB__ != 3 || __NEWLIB_MINOR__ != 3 || __NEWLIB_PATCHLEVEL__ != 0 +#error "MeshCore FS buffer fix: changed pinned newlib; review FILE layout" +#endif +static_assert(sizeof(void*) == 4 && offsetof(FILE, _bf) == 16 + && offsetof(__sbuf, _size) == 4, + "MeshCore FS buffer fix: changed pinned FILE layout"); +static_assert(std::is_same_bf._size), int>::value, + "MeshCore FS buffer fix: changed pinned buffer-size type"); +''' +ORIGINAL_METHOD = '''bool VFSFileImpl::setBufferSize(size_t size) +{ + if(_isDirectory || !_f) { + return 0; + } + int res = setvbuf(_f,NULL,_IOFBF,size); + return res == 0; +}''' +STRICT_METHOD = ORIGINAL_METHOD.replace( + " return res == 0;", + ''' // A successful newlib fallback may have selected a larger buffer. + // Preserve size==0's public choose-default behavior; bounded callers use + // nonzero sizes and must not start I/O unless their request was honored. + return res == 0 && (size == 0 || (_f->_bf._size > 0 + && static_cast(_f->_bf._size) == size));''') + + +def patched_vfs_source(source): + source = source.replace("\r\n", "\n") + digest = hashlib.sha256(source.encode("utf-8")).hexdigest() + if digest == PATCHED_VFS_SHA256: + return source + if (digest != PINNED_VFS_SHA256 or source.count(INCLUDE) != 1 + or source.count(ORIGINAL_METHOD) != 1): + raise RuntimeError("ESP32 FS buffer fix: changed pinned2.0.17 source; review SDK update") + patched = source.replace(INCLUDE, INCLUDE + LAYOUT_GUARD, 1).replace( + ORIGINAL_METHOD, STRICT_METHOD, 1) + if hashlib.sha256(patched.encode("utf-8")).hexdigest() != PATCHED_VFS_SHA256: + raise RuntimeError("ESP32 FS buffer fix: unexpected transform result") + return patched + + +def esp32_enabled(build_env): + # USB mode is intentionally irrelevant: UART V3 uses the same FS wrapper. + defines = {} + for definition in build_env.get("CPPDEFINES", []): + if isinstance(definition, (tuple, list)): + defines[str(definition[0])] = str(definition[1]) + else: + defines[str(definition)] = "1" + return any(defines.get(name) not in (None, "0") + for name in ("ESP32_PLATFORM", "ESP32")) + + +def require_pinned_framework(source): + version = source.parents[3] / "cores" / "esp32" / "esp_arduino_version.h" + try: + header = version.read_text(encoding="utf-8") + except OSError as error: + raise RuntimeError("ESP32 FS buffer fix: missing framework version header") from error + entries = re.findall(r"^#define ESP_ARDUINO_VERSION_(MAJOR|MINOR|PATCH)\s+(\d+)\s*$", + header, re.MULTILINE) + if (len(entries) != 3 or dict(entries) != {"MAJOR": "2", "MINOR": "0", "PATCH": "17"}): + raise RuntimeError("ESP32 FS buffer fix: changed/malformed pinned framework version") + + +def replace_vfs_source(build_env, node): + if not esp32_enabled(build_env): + return node + source = Path(node.srcnode().get_abspath()) + if source.parts[-4:] != SOURCE_SUFFIX: + return node + require_pinned_framework(source) + patched = patched_vfs_source(source.read_text(encoding="utf-8")) + destination = Path(build_env.subst("$BUILD_DIR")) / "patched-esp32-fs" / source.name + destination.parent.mkdir(parents=True, exist_ok=True) + if not destination.exists() or destination.read_text(encoding="utf-8") != patched: + destination.write_text(patched, encoding="utf-8") + # The copied source still includes the original FS library's private header. + build_env.AppendUnique(CPPPATH=[str(source.parent)]) + return build_env.File(str(destination)) + + +def install(build_env): + build_env.AddBuildMiddleware(replace_vfs_source, "*libraries*FS*src*vfs_api.cpp") + + +if "Import" in globals(): + Import("env") + install(env) diff --git a/test/fixtures/esp32_file_buffer/README.txt b/test/fixtures/esp32_file_buffer/README.txt new file mode 100644 index 00000000..584b2906 --- /dev/null +++ b/test/fixtures/esp32_file_buffer/README.txt @@ -0,0 +1,29 @@ +Pinned, byte-exact source fixtures for the build-local ESP File buffer check. + +vfs_api.cpp and esp_arduino_version.h: +Arduino-ESP32 2.0.17, revision dcc1105b0cf1322a437b354c336f2abf72b7e512. +https://github.com/espressif/arduino-esp32/blob/2.0.17/libraries/FS/src/vfs_api.cpp +https://github.com/espressif/arduino-esp32/blob/2.0.17/cores/esp32/esp_arduino_version.h +Both retain their complete upstream copyright/Apache-2.0 notice. +vfs_api.cpp SHA256 db3eb3fef8e59c0bafd01416ea18e2d5f36f38a9b180cd415ff3563a59fcfb65 +version header SHA256 b63e57731a24289b06eb96ded9e481ea719f4cde2ce7c41bd83c7646e0a6fe51 + +newlib_sys_reent.h and _newlib_version.h: +Unmodified installed Xtensa ESP32-S3 8.4.0+2021r2-patch5 toolchain headers, +newlib 3.3.0, as shipped with the pinned Arduino-ESP32 framework toolchain. +https://github.com/espressif/newlib-esp32/blob/esp-2021r2-patch5/newlib/libc/include/sys/reent.h +https://github.com/espressif/newlib-esp32/blob/esp-2021r2-patch5/COPYING.NEWLIB +All original header bytes/comments remain intact; _newlib_version.h is generated. +reent SHA256 33391d61a83ebbfb122e80514fbf7ed4d770556d8e8d46e029addbbf6077cd79 +version SHA256 93a397574a25b9c143cee7fcb86d8fc9a0c9ba418e4710a30d2d19ba1e4120db + +The actual ELF map resolves setvbuf/__sfvwrite_r/__srefill_r from the pinned +toolchain's no-rtti/libc.a. Its setvbuf allocation-fallback path may return zero +with a filesystem-sized buffer after the requested allocation failed. The +copied VFS method checks FILE::_bf._size before the caller's first read/write. +Compile-time newlib version/type/32-bit-prefix assertions fail closed when the +reviewed internal ABI changes. Native tests extract this exact FILE prefix and +model 32-bit pointer widths rather than borrowing the host libc's FILE layout. + +Backend buffer admission is not a wall-clock guarantee: SPIFFS metadata, +garbage collection, flush, close and publication still require hardware tests. diff --git a/test/fixtures/esp32_file_buffer/_newlib_version.h b/test/fixtures/esp32_file_buffer/_newlib_version.h new file mode 100644 index 00000000..f79befff --- /dev/null +++ b/test/fixtures/esp32_file_buffer/_newlib_version.h @@ -0,0 +1,12 @@ +/* _newlib_version.h. Generated from _newlib_version.hin by configure. */ +/* Version macros for internal and downstream use. */ +#ifndef _NEWLIB_VERSION_H__ +#define _NEWLIB_VERSION_H__ 1 + +#define _NEWLIB_VERSION "3.3.0" +#define __NEWLIB__ 3 +#define __NEWLIB_MINOR__ 3 +#define __NEWLIB_PATCHLEVEL__ 0 + +#endif /* !_NEWLIB_VERSION_H__ */ + diff --git a/test/fixtures/esp32_file_buffer/esp_arduino_version.h b/test/fixtures/esp32_file_buffer/esp_arduino_version.h new file mode 100644 index 00000000..755652f1 --- /dev/null +++ b/test/fixtures/esp32_file_buffer/esp_arduino_version.h @@ -0,0 +1,46 @@ +// Copyright 2019 Espressif Systems (Shanghai) PTE LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +/** Major version number (X.x.x) */ +#define ESP_ARDUINO_VERSION_MAJOR 2 +/** Minor version number (x.X.x) */ +#define ESP_ARDUINO_VERSION_MINOR 0 +/** Patch version number (x.x.X) */ +#define ESP_ARDUINO_VERSION_PATCH 17 + +/** + * Macro to convert ARDUINO version number into an integer + * + * To be used in comparisons, such as ESP_ARDUINO_VERSION >= ESP_ARDUINO_VERSION_VAL(2, 0, 0) + */ +#define ESP_ARDUINO_VERSION_VAL(major, minor, patch) ((major << 16) | (minor << 8) | (patch)) + +/** + * Current ARDUINO version, as an integer + * + * To be used in comparisons, such as ESP_ARDUINO_VERSION >= ESP_ARDUINO_VERSION_VAL(2, 0, 0) + */ +#define ESP_ARDUINO_VERSION ESP_ARDUINO_VERSION_VAL(ESP_ARDUINO_VERSION_MAJOR, \ + ESP_ARDUINO_VERSION_MINOR, \ + ESP_ARDUINO_VERSION_PATCH) + +#ifdef __cplusplus +} +#endif diff --git a/test/fixtures/esp32_file_buffer/newlib_sys_reent.h b/test/fixtures/esp32_file_buffer/newlib_sys_reent.h new file mode 100644 index 00000000..740b7921 --- /dev/null +++ b/test/fixtures/esp32_file_buffer/newlib_sys_reent.h @@ -0,0 +1,849 @@ +/* This header file provides the reentrancy. */ + +/* WARNING: All identifiers here must begin with an underscore. This file is + included by stdio.h and others and we therefore must only use identifiers + in the namespace allotted to us. */ + +#ifndef _SYS_REENT_H_ +#ifdef __cplusplus +extern "C" { +#endif +#define _SYS_REENT_H_ + +#include <_ansi.h> +#include +#include + +#define _NULL 0 + +#ifndef __Long +#if __LONG_MAX__ == 2147483647L +#define __Long long +typedef unsigned __Long __ULong; +#elif __INT_MAX__ == 2147483647 +#define __Long int +typedef unsigned __Long __ULong; +#endif +#endif + +#if !defined( __Long) +#include +#endif + +#ifndef __machine_flock_t_defined +#include +typedef _LOCK_RECURSIVE_T _flock_t; +#endif + +#ifndef __Long +#define __Long __int32_t +typedef __uint32_t __ULong; +#endif + +struct _reent; + +struct __locale_t; + +/* + * If _REENT_SMALL is defined, we make struct _reent as small as possible, + * by having nearly everything possible allocated at first use. + */ + +struct _Bigint +{ + struct _Bigint *_next; + int _k, _maxwds, _sign, _wds; + __ULong _x[1]; +}; + +/* needed by reentrant structure */ +struct __tm +{ + int __tm_sec; + int __tm_min; + int __tm_hour; + int __tm_mday; + int __tm_mon; + int __tm_year; + int __tm_wday; + int __tm_yday; + int __tm_isdst; +}; + +/* + * atexit() support. + */ + +#define _ATEXIT_SIZE 32 /* must be at least 32 to guarantee ANSI conformance */ + +struct _on_exit_args { + void * _fnargs[_ATEXIT_SIZE]; /* user fn args */ + void * _dso_handle[_ATEXIT_SIZE]; + /* Bitmask is set if user function takes arguments. */ + __ULong _fntypes; /* type of exit routine - + Must have at least _ATEXIT_SIZE bits */ + /* Bitmask is set if function was registered via __cxa_atexit. */ + __ULong _is_cxa; +}; + +#ifdef _REENT_SMALL +struct _atexit { + struct _atexit *_next; /* next in list */ + int _ind; /* next index in this table */ + void (*_fns[_ATEXIT_SIZE])(void); /* the table itself */ + struct _on_exit_args * _on_exit_args_ptr; +}; +# define _ATEXIT_INIT {_NULL, 0, {_NULL}, _NULL} +#else +struct _atexit { + struct _atexit *_next; /* next in list */ + int _ind; /* next index in this table */ + /* Some entries may already have been called, and will be NULL. */ + void (*_fns[_ATEXIT_SIZE])(void); /* the table itself */ + struct _on_exit_args _on_exit_args; +}; +# define _ATEXIT_INIT {_NULL, 0, {_NULL}, {{_NULL}, {_NULL}, 0, 0}} +#endif + +#ifdef _REENT_GLOBAL_ATEXIT +# define _REENT_INIT_ATEXIT +#else +# define _REENT_INIT_ATEXIT \ + _NULL, _ATEXIT_INIT, +#endif + +/* + * Stdio buffers. + * + * This and __FILE are defined here because we need them for struct _reent, + * but we don't want stdio.h included when stdlib.h is. + */ + +struct __sbuf { + unsigned char *_base; + int _size; +}; + +/* + * Stdio state variables. + * + * The following always hold: + * + * if (_flags&(__SLBF|__SWR)) == (__SLBF|__SWR), + * _lbfsize is -_bf._size, else _lbfsize is 0 + * if _flags&__SRD, _w is 0 + * if _flags&__SWR, _r is 0 + * + * This ensures that the getc and putc macros (or inline functions) never + * try to write or read from a file that is in `read' or `write' mode. + * (Moreover, they can, and do, automatically switch from read mode to + * write mode, and back, on "r+" and "w+" files.) + * + * _lbfsize is used only to make the inline line-buffered output stream + * code as compact as possible. + * + * _ub, _up, and _ur are used when ungetc() pushes back more characters + * than fit in the current _bf, or when ungetc() pushes back a character + * that does not match the previous one in _bf. When this happens, + * _ub._base becomes non-nil (i.e., a stream has ungetc() data iff + * _ub._base!=NULL) and _up and _ur save the current values of _p and _r. + */ + +#if defined(_REENT_SMALL) && !defined(_REENT_GLOBAL_STDIO_STREAMS) +/* + * struct __sFILE_fake is the start of a struct __sFILE, with only the + * minimal fields allocated. In __sinit() we really allocate the 3 + * standard streams, etc., and point away from this fake. + */ +struct __sFILE_fake { + unsigned char *_p; /* current position in (some) buffer */ + int _r; /* read space left for getc() */ + int _w; /* write space left for putc() */ + short _flags; /* flags, below; this FILE is free if 0 */ + short _file; /* fileno, if Unix descriptor, else -1 */ + struct __sbuf _bf; /* the buffer (at least 1 byte, if !NULL) */ + int _lbfsize; /* 0 or -_bf._size, for inline putc */ + + struct _reent *_data; +}; + +/* Following is needed both in libc/stdio and libc/stdlib so we put it + * here instead of libc/stdio/local.h where it was previously. */ + +extern void __sinit (struct _reent *); + +# define _REENT_SMALL_CHECK_INIT(ptr) \ + do \ + { \ + if ((ptr) && !(ptr)->__sdidinit) \ + __sinit (ptr); \ + } \ + while (0) +#else /* _REENT_SMALL && !_REENT_GLOBAL_STDIO_STREAMS */ +# define _REENT_SMALL_CHECK_INIT(ptr) /* nothing */ +#endif /* _REENT_SMALL && !_REENT_GLOBAL_STDIO_STREAMS */ + +struct __sFILE { + unsigned char *_p; /* current position in (some) buffer */ + int _r; /* read space left for getc() */ + int _w; /* write space left for putc() */ + short _flags; /* flags, below; this FILE is free if 0 */ + short _file; /* fileno, if Unix descriptor, else -1 */ + struct __sbuf _bf; /* the buffer (at least 1 byte, if !NULL) */ + int _lbfsize; /* 0 or -_bf._size, for inline putc */ + +#ifdef _REENT_SMALL + struct _reent *_data; +#endif + + /* operations */ + void * _cookie; /* cookie passed to io functions */ + + _READ_WRITE_RETURN_TYPE (*_read) (struct _reent *, void *, + char *, _READ_WRITE_BUFSIZE_TYPE); + _READ_WRITE_RETURN_TYPE (*_write) (struct _reent *, void *, + const char *, + _READ_WRITE_BUFSIZE_TYPE); + _fpos_t (*_seek) (struct _reent *, void *, _fpos_t, int); + int (*_close) (struct _reent *, void *); + + /* separate buffer for long sequences of ungetc() */ + struct __sbuf _ub; /* ungetc buffer */ + unsigned char *_up; /* saved _p when _p is doing ungetc data */ + int _ur; /* saved _r when _r is counting ungetc data */ + + /* tricks to meet minimum requirements even when malloc() fails */ + unsigned char _ubuf[3]; /* guarantee an ungetc() buffer */ + unsigned char _nbuf[1]; /* guarantee a getc() buffer */ + + /* separate buffer for fgetline() when line crosses buffer boundary */ + struct __sbuf _lb; /* buffer for fgetline() */ + + /* Unix stdio files get aligned to block boundaries on fseek() */ + int _blksize; /* stat.st_blksize (may be != _bf._size) */ + _off_t _offset; /* current lseek offset */ + +#ifndef _REENT_SMALL + struct _reent *_data; /* Here for binary compatibility? Remove? */ +#endif + +#ifndef __SINGLE_THREAD__ + _flock_t _lock; /* for thread-safety locking */ +#endif + _mbstate_t _mbstate; /* for wide char stdio functions. */ + int _flags2; /* for future use */ +}; + +#ifdef __CUSTOM_FILE_IO__ + +/* Get custom _FILE definition. */ +#include + +#else /* !__CUSTOM_FILE_IO__ */ +#ifdef __LARGE64_FILES +struct __sFILE64 { + unsigned char *_p; /* current position in (some) buffer */ + int _r; /* read space left for getc() */ + int _w; /* write space left for putc() */ + short _flags; /* flags, below; this FILE is free if 0 */ + short _file; /* fileno, if Unix descriptor, else -1 */ + struct __sbuf _bf; /* the buffer (at least 1 byte, if !NULL) */ + int _lbfsize; /* 0 or -_bf._size, for inline putc */ + + struct _reent *_data; + + /* operations */ + void * _cookie; /* cookie passed to io functions */ + + _READ_WRITE_RETURN_TYPE (*_read) (struct _reent *, void *, + char *, _READ_WRITE_BUFSIZE_TYPE); + _READ_WRITE_RETURN_TYPE (*_write) (struct _reent *, void *, + const char *, + _READ_WRITE_BUFSIZE_TYPE); + _fpos_t (*_seek) (struct _reent *, void *, _fpos_t, int); + int (*_close) (struct _reent *, void *); + + /* separate buffer for long sequences of ungetc() */ + struct __sbuf _ub; /* ungetc buffer */ + unsigned char *_up; /* saved _p when _p is doing ungetc data */ + int _ur; /* saved _r when _r is counting ungetc data */ + + /* tricks to meet minimum requirements even when malloc() fails */ + unsigned char _ubuf[3]; /* guarantee an ungetc() buffer */ + unsigned char _nbuf[1]; /* guarantee a getc() buffer */ + + /* separate buffer for fgetline() when line crosses buffer boundary */ + struct __sbuf _lb; /* buffer for fgetline() */ + + /* Unix stdio files get aligned to block boundaries on fseek() */ + int _blksize; /* stat.st_blksize (may be != _bf._size) */ + int _flags2; /* for future use */ + + _off64_t _offset; /* current lseek offset */ + _fpos64_t (*_seek64) (struct _reent *, void *, _fpos64_t, int); + +#ifndef __SINGLE_THREAD__ + _flock_t _lock; /* for thread-safety locking */ +#endif + _mbstate_t _mbstate; /* for wide char stdio functions. */ +}; +typedef struct __sFILE64 __FILE; +#else +typedef struct __sFILE __FILE; +#endif /* __LARGE64_FILES */ +#endif /* !__CUSTOM_FILE_IO__ */ + +struct _glue +{ + struct _glue *_next; + int _niobs; + __FILE *_iobs; +}; + +/* + * rand48 family support + * + * Copyright (c) 1993 Martin Birgmeier + * All rights reserved. + * + * You may redistribute unmodified or modified versions of this source + * code provided that the above copyright notice and this and the + * following conditions are retained. + * + * This software is provided ``as is'', and comes with no warranties + * of any kind. I shall in no event be liable for anything that happens + * to anyone/anything when using this software. + */ +#define _RAND48_SEED_0 (0x330e) +#define _RAND48_SEED_1 (0xabcd) +#define _RAND48_SEED_2 (0x1234) +#define _RAND48_MULT_0 (0xe66d) +#define _RAND48_MULT_1 (0xdeec) +#define _RAND48_MULT_2 (0x0005) +#define _RAND48_ADD (0x000b) +struct _rand48 { + unsigned short _seed[3]; + unsigned short _mult[3]; + unsigned short _add; +#ifdef _REENT_SMALL + /* Put this in here as well, for good luck. */ + __extension__ unsigned long long _rand_next; +#endif +}; + +/* How big the some arrays are. */ +#define _REENT_EMERGENCY_SIZE 25 +#define _REENT_ASCTIME_SIZE 26 +#define _REENT_SIGNAL_SIZE 24 + +/* + * struct _reent + * + * This structure contains *all* globals needed by the library. + * It's raison d'etre is to facilitate threads by making all library routines + * reentrant. IE: All state information is contained here. + */ + +#ifdef _REENT_SMALL + +struct _mprec +{ + /* used by mprec routines */ + struct _Bigint *_result; + int _result_k; + struct _Bigint *_p5s; + struct _Bigint **_freelist; +}; + + +struct _misc_reent +{ + /* miscellaneous reentrant data */ + char *_strtok_last; + _mbstate_t _mblen_state; + _mbstate_t _wctomb_state; + _mbstate_t _mbtowc_state; + char _l64a_buf[8]; + int _getdate_err; + _mbstate_t _mbrlen_state; + _mbstate_t _mbrtowc_state; + _mbstate_t _mbsrtowcs_state; + _mbstate_t _wcrtomb_state; + _mbstate_t _wcsrtombs_state; +}; + +/* This version of _reent is laid out with "int"s in pairs, to help + * ports with 16-bit int's but 32-bit pointers, align nicely. */ +struct _reent +{ + /* As an exception to the above put _errno first for binary + compatibility with non _REENT_SMALL targets. */ + int _errno; /* local copy of errno */ + + /* FILE is a big struct and may change over time. To try to achieve binary + compatibility with future versions, put stdin,stdout,stderr here. + These are pointers into member __sf defined below. */ + __FILE *_stdin, *_stdout, *_stderr; /* XXX */ + + int _inc; /* used by tmpnam */ + + char *_emergency; + + int __sdidinit; /* 1 means stdio has been init'd */ + + int _unspecified_locale_info; /* unused, reserved for locale stuff */ + struct __locale_t *_locale;/* per-thread locale */ + + struct _mprec *_mp; + + void (*__cleanup) (struct _reent *); + + int _gamma_signgam; + + /* used by some fp conversion routines */ + int _cvtlen; /* should be size_t */ + char *_cvtbuf; + + struct _rand48 *_r48; + struct __tm *_localtime_buf; + char *_asctime_buf; + + /* signal info */ + void (**_sig_func)(int); + +# ifndef _REENT_GLOBAL_ATEXIT + /* atexit stuff */ + struct _atexit *_atexit; + struct _atexit _atexit0; +# endif + + struct _glue __sglue; /* root of glue chain */ + __FILE *__sf; /* file descriptors */ + struct _misc_reent *_misc; /* strtok, multibyte states */ + char *_signal_buf; /* strsignal */ +}; + +#ifdef _REENT_GLOBAL_STDIO_STREAMS +extern __FILE __sf[3]; + +# define _REENT_INIT(var) \ + { 0, \ + &__sf[0], \ + &__sf[1], \ + &__sf[2], \ + 0, \ + _NULL, \ + 0, \ + 0, \ + _NULL, \ + _NULL, \ + _NULL, \ + 0, \ + 0, \ + _NULL, \ + _NULL, \ + _NULL, \ + _NULL, \ + _NULL, \ + _REENT_INIT_ATEXIT \ + {_NULL, 0, _NULL}, \ + _NULL, \ + _NULL, \ + _NULL \ + } + +#define _REENT_INIT_PTR_ZEROED(var) \ + { (var)->_stdin = &__sf[0]; \ + (var)->_stdout = &__sf[1]; \ + (var)->_stderr = &__sf[2]; \ + } + +#else /* _REENT_GLOBAL_STDIO_STREAMS */ + +extern const struct __sFILE_fake __sf_fake_stdin; +extern const struct __sFILE_fake __sf_fake_stdout; +extern const struct __sFILE_fake __sf_fake_stderr; + +# define _REENT_INIT(var) \ + { 0, \ + (__FILE *)&__sf_fake_stdin, \ + (__FILE *)&__sf_fake_stdout, \ + (__FILE *)&__sf_fake_stderr, \ + 0, \ + _NULL, \ + 0, \ + 0, \ + _NULL, \ + _NULL, \ + _NULL, \ + 0, \ + 0, \ + _NULL, \ + _NULL, \ + _NULL, \ + _NULL, \ + _NULL, \ + _REENT_INIT_ATEXIT \ + {_NULL, 0, _NULL}, \ + _NULL, \ + _NULL, \ + _NULL \ + } + +#define _REENT_INIT_PTR_ZEROED(var) \ + { (var)->_stdin = (__FILE *)&__sf_fake_stdin; \ + (var)->_stdout = (__FILE *)&__sf_fake_stdout; \ + (var)->_stderr = (__FILE *)&__sf_fake_stderr; \ + } + +#endif /* _REENT_GLOBAL_STDIO_STREAMS */ + +/* Specify how to handle reent_check malloc failures. */ +#ifdef _REENT_CHECK_VERIFY +#include +#define __reent_assert(x) ((x) ? (void)0 : __assert_func(__FILE__, __LINE__, (char *)0, "REENT malloc succeeded")) +#else +#define __reent_assert(x) ((void)0) +#endif + +#ifdef __CUSTOM_FILE_IO__ +#error Custom FILE I/O and _REENT_SMALL not currently supported. +#endif + +/* Generic _REENT check macro. */ +#define _REENT_CHECK(var, what, type, size, init) do { \ + struct _reent *_r = (var); \ + if (_r->what == NULL) { \ + _r->what = (type)malloc(size); \ + __reent_assert(_r->what); \ + init; \ + } \ +} while (0) + +#define _REENT_CHECK_TM(var) \ + _REENT_CHECK(var, _localtime_buf, struct __tm *, sizeof *((var)->_localtime_buf), \ + /* nothing */) + +#define _REENT_CHECK_ASCTIME_BUF(var) \ + _REENT_CHECK(var, _asctime_buf, char *, _REENT_ASCTIME_SIZE, \ + memset((var)->_asctime_buf, 0, _REENT_ASCTIME_SIZE)) + +/* Handle the dynamically allocated rand48 structure. */ +#define _REENT_INIT_RAND48(var) do { \ + struct _reent *_r = (var); \ + _r->_r48->_seed[0] = _RAND48_SEED_0; \ + _r->_r48->_seed[1] = _RAND48_SEED_1; \ + _r->_r48->_seed[2] = _RAND48_SEED_2; \ + _r->_r48->_mult[0] = _RAND48_MULT_0; \ + _r->_r48->_mult[1] = _RAND48_MULT_1; \ + _r->_r48->_mult[2] = _RAND48_MULT_2; \ + _r->_r48->_add = _RAND48_ADD; \ + _r->_r48->_rand_next = 1; \ +} while (0) +#define _REENT_CHECK_RAND48(var) \ + _REENT_CHECK(var, _r48, struct _rand48 *, sizeof *((var)->_r48), _REENT_INIT_RAND48((var))) + +#define _REENT_INIT_MP(var) do { \ + struct _reent *_r = (var); \ + _r->_mp->_result_k = 0; \ + _r->_mp->_result = _r->_mp->_p5s = _NULL; \ + _r->_mp->_freelist = _NULL; \ +} while (0) +#define _REENT_CHECK_MP(var) \ + _REENT_CHECK(var, _mp, struct _mprec *, sizeof *((var)->_mp), _REENT_INIT_MP(var)) + +#define _REENT_CHECK_EMERGENCY(var) \ + _REENT_CHECK(var, _emergency, char *, _REENT_EMERGENCY_SIZE, /* nothing */) + +#define _REENT_INIT_MISC(var) do { \ + struct _reent *_r = (var); \ + _r->_misc->_strtok_last = _NULL; \ + _r->_misc->_mblen_state.__count = 0; \ + _r->_misc->_mblen_state.__value.__wch = 0; \ + _r->_misc->_wctomb_state.__count = 0; \ + _r->_misc->_wctomb_state.__value.__wch = 0; \ + _r->_misc->_mbtowc_state.__count = 0; \ + _r->_misc->_mbtowc_state.__value.__wch = 0; \ + _r->_misc->_mbrlen_state.__count = 0; \ + _r->_misc->_mbrlen_state.__value.__wch = 0; \ + _r->_misc->_mbrtowc_state.__count = 0; \ + _r->_misc->_mbrtowc_state.__value.__wch = 0; \ + _r->_misc->_mbsrtowcs_state.__count = 0; \ + _r->_misc->_mbsrtowcs_state.__value.__wch = 0; \ + _r->_misc->_wcrtomb_state.__count = 0; \ + _r->_misc->_wcrtomb_state.__value.__wch = 0; \ + _r->_misc->_wcsrtombs_state.__count = 0; \ + _r->_misc->_wcsrtombs_state.__value.__wch = 0; \ + _r->_misc->_l64a_buf[0] = '\0'; \ + _r->_misc->_getdate_err = 0; \ +} while (0) +#define _REENT_CHECK_MISC(var) \ + _REENT_CHECK(var, _misc, struct _misc_reent *, sizeof *((var)->_misc), _REENT_INIT_MISC(var)) + +#define _REENT_CHECK_SIGNAL_BUF(var) \ + _REENT_CHECK(var, _signal_buf, char *, _REENT_SIGNAL_SIZE, /* nothing */) + +#define _REENT_SIGNGAM(ptr) ((ptr)->_gamma_signgam) +#define _REENT_RAND_NEXT(ptr) ((ptr)->_r48->_rand_next) +#define _REENT_RAND48_SEED(ptr) ((ptr)->_r48->_seed) +#define _REENT_RAND48_MULT(ptr) ((ptr)->_r48->_mult) +#define _REENT_RAND48_ADD(ptr) ((ptr)->_r48->_add) +#define _REENT_MP_RESULT(ptr) ((ptr)->_mp->_result) +#define _REENT_MP_RESULT_K(ptr) ((ptr)->_mp->_result_k) +#define _REENT_MP_P5S(ptr) ((ptr)->_mp->_p5s) +#define _REENT_MP_FREELIST(ptr) ((ptr)->_mp->_freelist) +#define _REENT_ASCTIME_BUF(ptr) ((ptr)->_asctime_buf) +#define _REENT_TM(ptr) ((ptr)->_localtime_buf) +#define _REENT_EMERGENCY(ptr) ((ptr)->_emergency) +#define _REENT_STRTOK_LAST(ptr) ((ptr)->_misc->_strtok_last) +#define _REENT_MBLEN_STATE(ptr) ((ptr)->_misc->_mblen_state) +#define _REENT_MBTOWC_STATE(ptr)((ptr)->_misc->_mbtowc_state) +#define _REENT_WCTOMB_STATE(ptr)((ptr)->_misc->_wctomb_state) +#define _REENT_MBRLEN_STATE(ptr) ((ptr)->_misc->_mbrlen_state) +#define _REENT_MBRTOWC_STATE(ptr) ((ptr)->_misc->_mbrtowc_state) +#define _REENT_MBSRTOWCS_STATE(ptr) ((ptr)->_misc->_mbsrtowcs_state) +#define _REENT_WCRTOMB_STATE(ptr) ((ptr)->_misc->_wcrtomb_state) +#define _REENT_WCSRTOMBS_STATE(ptr) ((ptr)->_misc->_wcsrtombs_state) +#define _REENT_L64A_BUF(ptr) ((ptr)->_misc->_l64a_buf) +#define _REENT_GETDATE_ERR_P(ptr) (&((ptr)->_misc->_getdate_err)) +#define _REENT_SIGNAL_BUF(ptr) ((ptr)->_signal_buf) + +#else /* !_REENT_SMALL */ + +struct _reent +{ + int _errno; /* local copy of errno */ + + /* FILE is a big struct and may change over time. To try to achieve binary + compatibility with future versions, put stdin,stdout,stderr here. + These are pointers into member __sf defined below. */ + __FILE *_stdin, *_stdout, *_stderr; + + int _inc; /* used by tmpnam */ + char _emergency[_REENT_EMERGENCY_SIZE]; + + /* TODO */ + int _unspecified_locale_info; /* unused, reserved for locale stuff */ + struct __locale_t *_locale;/* per-thread locale */ + + int __sdidinit; /* 1 means stdio has been init'd */ + + void (*__cleanup) (struct _reent *); + + /* used by mprec routines */ + struct _Bigint *_result; + int _result_k; + struct _Bigint *_p5s; + struct _Bigint **_freelist; + + /* used by some fp conversion routines */ + int _cvtlen; /* should be size_t */ + char *_cvtbuf; + + union + { + struct + { + unsigned int _unused_rand; + char * _strtok_last; + char _asctime_buf[_REENT_ASCTIME_SIZE]; + struct __tm _localtime_buf; + int _gamma_signgam; + __extension__ unsigned long long _rand_next; + struct _rand48 _r48; + _mbstate_t _mblen_state; + _mbstate_t _mbtowc_state; + _mbstate_t _wctomb_state; + char _l64a_buf[8]; + char _signal_buf[_REENT_SIGNAL_SIZE]; + int _getdate_err; + _mbstate_t _mbrlen_state; + _mbstate_t _mbrtowc_state; + _mbstate_t _mbsrtowcs_state; + _mbstate_t _wcrtomb_state; + _mbstate_t _wcsrtombs_state; + int _h_errno; + } _reent; + /* Two next two fields were once used by malloc. They are no longer + used. They are used to preserve the space used before so as to + allow addition of new reent fields and keep binary compatibility. */ + struct + { +#define _N_LISTS 30 + unsigned char * _nextf[_N_LISTS]; + unsigned int _nmalloc[_N_LISTS]; + } _unused; + } _new; + +# ifndef _REENT_GLOBAL_ATEXIT + /* atexit stuff */ + struct _atexit *_atexit; /* points to head of LIFO stack */ + struct _atexit _atexit0; /* one guaranteed table, required by ANSI */ +# endif + + /* signal info */ + void (**(_sig_func))(int); + + /* These are here last so that __FILE can grow without changing the offsets + of the above members (on the off chance that future binary compatibility + would be broken otherwise). */ + struct _glue __sglue; /* root of glue chain */ +# ifndef _REENT_GLOBAL_STDIO_STREAMS + __FILE __sf[3]; /* first three file descriptors */ +# endif +}; + +#ifdef _REENT_GLOBAL_STDIO_STREAMS +extern __FILE __sf[3]; +#define _REENT_STDIO_STREAM(var, index) &__sf[index] +#else +#define _REENT_STDIO_STREAM(var, index) &(var)->__sf[index] +#endif + +#define _REENT_INIT(var) \ + { 0, \ + _REENT_STDIO_STREAM(&(var), 0), \ + _REENT_STDIO_STREAM(&(var), 1), \ + _REENT_STDIO_STREAM(&(var), 2), \ + 0, \ + "", \ + 0, \ + _NULL, \ + 0, \ + _NULL, \ + _NULL, \ + 0, \ + _NULL, \ + _NULL, \ + 0, \ + _NULL, \ + { \ + { \ + 0, \ + _NULL, \ + "", \ + {0, 0, 0, 0, 0, 0, 0, 0, 0}, \ + 0, \ + 1, \ + { \ + {_RAND48_SEED_0, _RAND48_SEED_1, _RAND48_SEED_2}, \ + {_RAND48_MULT_0, _RAND48_MULT_1, _RAND48_MULT_2}, \ + _RAND48_ADD \ + }, \ + {0, {0}}, \ + {0, {0}}, \ + {0, {0}}, \ + "", \ + "", \ + 0, \ + {0, {0}}, \ + {0, {0}}, \ + {0, {0}}, \ + {0, {0}}, \ + {0, {0}} \ + } \ + }, \ + _REENT_INIT_ATEXIT \ + _NULL, \ + {_NULL, 0, _NULL} \ + } + +#define _REENT_INIT_PTR_ZEROED(var) \ + { (var)->_stdin = _REENT_STDIO_STREAM(var, 0); \ + (var)->_stdout = _REENT_STDIO_STREAM(var, 1); \ + (var)->_stderr = _REENT_STDIO_STREAM(var, 2); \ + (var)->_new._reent._rand_next = 1; \ + (var)->_new._reent._r48._seed[0] = _RAND48_SEED_0; \ + (var)->_new._reent._r48._seed[1] = _RAND48_SEED_1; \ + (var)->_new._reent._r48._seed[2] = _RAND48_SEED_2; \ + (var)->_new._reent._r48._mult[0] = _RAND48_MULT_0; \ + (var)->_new._reent._r48._mult[1] = _RAND48_MULT_1; \ + (var)->_new._reent._r48._mult[2] = _RAND48_MULT_2; \ + (var)->_new._reent._r48._add = _RAND48_ADD; \ + } + +#define _REENT_CHECK_RAND48(ptr) /* nothing */ +#define _REENT_CHECK_MP(ptr) /* nothing */ +#define _REENT_CHECK_TM(ptr) /* nothing */ +#define _REENT_CHECK_ASCTIME_BUF(ptr) /* nothing */ +#define _REENT_CHECK_EMERGENCY(ptr) /* nothing */ +#define _REENT_CHECK_MISC(ptr) /* nothing */ +#define _REENT_CHECK_SIGNAL_BUF(ptr) /* nothing */ + +#define _REENT_SIGNGAM(ptr) ((ptr)->_new._reent._gamma_signgam) +#define _REENT_RAND_NEXT(ptr) ((ptr)->_new._reent._rand_next) +#define _REENT_RAND48_SEED(ptr) ((ptr)->_new._reent._r48._seed) +#define _REENT_RAND48_MULT(ptr) ((ptr)->_new._reent._r48._mult) +#define _REENT_RAND48_ADD(ptr) ((ptr)->_new._reent._r48._add) +#define _REENT_MP_RESULT(ptr) ((ptr)->_result) +#define _REENT_MP_RESULT_K(ptr) ((ptr)->_result_k) +#define _REENT_MP_P5S(ptr) ((ptr)->_p5s) +#define _REENT_MP_FREELIST(ptr) ((ptr)->_freelist) +#define _REENT_ASCTIME_BUF(ptr) ((ptr)->_new._reent._asctime_buf) +#define _REENT_TM(ptr) (&(ptr)->_new._reent._localtime_buf) +#define _REENT_EMERGENCY(ptr) ((ptr)->_emergency) +#define _REENT_STRTOK_LAST(ptr) ((ptr)->_new._reent._strtok_last) +#define _REENT_MBLEN_STATE(ptr) ((ptr)->_new._reent._mblen_state) +#define _REENT_MBTOWC_STATE(ptr)((ptr)->_new._reent._mbtowc_state) +#define _REENT_WCTOMB_STATE(ptr)((ptr)->_new._reent._wctomb_state) +#define _REENT_MBRLEN_STATE(ptr)((ptr)->_new._reent._mbrlen_state) +#define _REENT_MBRTOWC_STATE(ptr)((ptr)->_new._reent._mbrtowc_state) +#define _REENT_MBSRTOWCS_STATE(ptr)((ptr)->_new._reent._mbsrtowcs_state) +#define _REENT_WCRTOMB_STATE(ptr)((ptr)->_new._reent._wcrtomb_state) +#define _REENT_WCSRTOMBS_STATE(ptr)((ptr)->_new._reent._wcsrtombs_state) +#define _REENT_L64A_BUF(ptr) ((ptr)->_new._reent._l64a_buf) +#define _REENT_SIGNAL_BUF(ptr) ((ptr)->_new._reent._signal_buf) +#define _REENT_GETDATE_ERR_P(ptr) (&((ptr)->_new._reent._getdate_err)) + +#endif /* !_REENT_SMALL */ + +#define _REENT_INIT_PTR(var) \ + { memset((var), 0, sizeof(*(var))); \ + _REENT_INIT_PTR_ZEROED(var); \ + } + +/* This value is used in stdlib/misc.c. reent/reent.c has to know it + as well to make sure the freelist is correctly free'd. Therefore + we define it here, rather than in stdlib/misc.c, as before. */ +#define _Kmax (sizeof (size_t) << 3) + +/* + * All references to struct _reent are via this pointer. + * Internally, newlib routines that need to reference it should use _REENT. + */ + +#ifndef __ATTRIBUTE_IMPURE_PTR__ +#define __ATTRIBUTE_IMPURE_PTR__ +#endif + +#if !defined(__DYNAMIC_REENT__) || defined(__SINGLE_THREAD__) +extern struct _reent *_impure_ptr __ATTRIBUTE_IMPURE_PTR__; +#endif +extern struct _reent * _global_impure_ptr __ATTRIBUTE_IMPURE_PTR__; + +void _reclaim_reent (struct _reent *); + +/* #define _REENT_ONLY define this to get only reentrant routines */ + +#if defined(__DYNAMIC_REENT__) && !defined(__SINGLE_THREAD__) +#ifndef __getreent + struct _reent * __getreent (void); +#endif +# define _REENT (__getreent()) +#else /* __SINGLE_THREAD__ || !__DYNAMIC_REENT__ */ +# define _REENT _impure_ptr +#endif /* __SINGLE_THREAD__ || !__DYNAMIC_REENT__ */ + +#define _GLOBAL_REENT _global_impure_ptr + +#ifdef _REENT_GLOBAL_ATEXIT +extern struct _atexit *_global_atexit; /* points to head of LIFO stack */ +# define _GLOBAL_ATEXIT _global_atexit +#else +# define _GLOBAL_ATEXIT (_GLOBAL_REENT->_atexit) +#endif + +#ifdef __cplusplus +} +#endif +#endif /* _SYS_REENT_H_ */ diff --git a/test/fixtures/esp32_file_buffer/vfs_api.cpp b/test/fixtures/esp32_file_buffer/vfs_api.cpp new file mode 100644 index 00000000..1dd8da94 --- /dev/null +++ b/test/fixtures/esp32_file_buffer/vfs_api.cpp @@ -0,0 +1,575 @@ +// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "vfs_api.h" + +using namespace fs; + +#define DEFAULT_FILE_BUFFER_SIZE 4096 + +FileImplPtr VFSImpl::open(const char* fpath, const char* mode, const bool create) +{ + if(!_mountpoint) { + log_e("File system is not mounted"); + return FileImplPtr(); + } + + if(!fpath || fpath[0] != '/') { + log_e("%s does not start with /", fpath); + return FileImplPtr(); + } + + char * temp = (char *)malloc(strlen(fpath)+strlen(_mountpoint)+2); + if(!temp) { + log_e("malloc failed"); + return FileImplPtr(); + } + + strcpy(temp, _mountpoint); + strcat(temp, fpath); + + struct stat st; + //file found + if(!stat(temp, &st)) { + free(temp); + if (S_ISREG(st.st_mode) || S_ISDIR(st.st_mode)) { + return std::make_shared(this, fpath, mode); + } + log_e("%s has wrong mode 0x%08X", fpath, st.st_mode); + return FileImplPtr(); + } + + //try to open this as directory (might be mount point) + DIR * d = opendir(temp); + if(d) { + closedir(d); + free(temp); + return std::make_shared(this, fpath, mode); + } + + //file not found but mode permits file creation without folder creation + if((mode && mode[0] != 'r') && (!create)){ + free(temp); + return std::make_shared(this, fpath, mode); + } + + ////file not found but mode permits file creation and folder creation + if((mode && mode[0] != 'r') && create){ + + char *token; + char *folder = (char *)malloc(strlen(fpath)); + + int start_index = 0; + int end_index = 0; + + token = strchr(fpath+1,'/'); + end_index = (token-fpath); + + while (token != NULL) + { + memcpy(folder,fpath + start_index, end_index-start_index); + folder[end_index-start_index] = '\0'; + + if(!VFSImpl::mkdir(folder)) + { + log_e("Creating folder: %s failed!",folder); + return FileImplPtr(); + } + + token=strchr(token+1,'/'); + if(token != NULL) + { + end_index = (token-fpath); + memset(folder, 0, strlen(folder)); + } + + } + + free(folder); + free(temp); + return std::make_shared(this, fpath, mode); + + } + + log_e("%s does not exist, no permits for creation", temp); + free(temp); + return FileImplPtr(); +} + +bool VFSImpl::exists(const char* fpath) +{ + if(!_mountpoint) { + log_e("File system is not mounted"); + return false; + } + + VFSFileImpl f(this, fpath, "r"); + if(f) { + f.close(); + return true; + } + return false; +} + +bool VFSImpl::rename(const char* pathFrom, const char* pathTo) +{ + if(!_mountpoint) { + log_e("File system is not mounted"); + return false; + } + + if(!pathFrom || pathFrom[0] != '/' || !pathTo || pathTo[0] != '/') { + log_e("bad arguments"); + return false; + } + if(!exists(pathFrom)) { + log_e("%s does not exists", pathFrom); + return false; + } + size_t mountpointLen = strlen(_mountpoint); + char * temp1 = (char *)malloc(strlen(pathFrom)+mountpointLen+1); + if(!temp1) { + log_e("malloc failed"); + return false; + } + char * temp2 = (char *)malloc(strlen(pathTo)+mountpointLen+1); + if(!temp2) { + free(temp1); + log_e("malloc failed"); + return false; + } + + strcpy(temp1, _mountpoint); + strcat(temp1, pathFrom); + + strcpy(temp2, _mountpoint); + strcat(temp2, pathTo); + + auto rc = ::rename(temp1, temp2); + free(temp1); + free(temp2); + return rc == 0; +} + +bool VFSImpl::remove(const char* fpath) +{ + if(!_mountpoint) { + log_e("File system is not mounted"); + return false; + } + + if(!fpath || fpath[0] != '/') { + log_e("bad arguments"); + return false; + } + + VFSFileImpl f(this, fpath, "r"); + if(!f || f.isDirectory()) { + if(f) { + f.close(); + } + log_e("%s does not exists or is directory", fpath); + return false; + } + f.close(); + + char * temp = (char *)malloc(strlen(fpath)+strlen(_mountpoint)+1); + if(!temp) { + log_e("malloc failed"); + return false; + } + + strcpy(temp, _mountpoint); + strcat(temp, fpath); + + auto rc = unlink(temp); + free(temp); + return rc == 0; +} + +bool VFSImpl::mkdir(const char *fpath) +{ + if(!_mountpoint) { + log_e("File system is not mounted"); + return false; + } + + VFSFileImpl f(this, fpath, "r"); + if(f && f.isDirectory()) { + f.close(); + //log_w("%s already exists", fpath); + return true; + } else if(f) { + f.close(); + log_e("%s is a file", fpath); + return false; + } + + char * temp = (char *)malloc(strlen(fpath)+strlen(_mountpoint)+1); + if(!temp) { + log_e("malloc failed"); + return false; + } + + strcpy(temp, _mountpoint); + strcat(temp, fpath); + + auto rc = ::mkdir(temp, ACCESSPERMS); + free(temp); + return rc == 0; +} + +bool VFSImpl::rmdir(const char *fpath) +{ + if(!_mountpoint) { + log_e("File system is not mounted"); + return false; + } + + if (strcmp(_mountpoint, "/spiffs") == 0) { + log_e("rmdir is unnecessary in SPIFFS"); + return false; + } + + VFSFileImpl f(this, fpath, "r"); + if(!f || !f.isDirectory()) { + if(f) { + f.close(); + } + log_e("%s does not exists or is a file", fpath); + return false; + } + f.close(); + + char * temp = (char *)malloc(strlen(fpath)+strlen(_mountpoint)+1); + if(!temp) { + log_e("malloc failed"); + return false; + } + + strcpy(temp, _mountpoint); + strcat(temp, fpath); + + auto rc = ::rmdir(temp); + free(temp); + return rc == 0; +} + + + + +VFSFileImpl::VFSFileImpl(VFSImpl* fs, const char* fpath, const char* mode) + : _fs(fs) + , _f(NULL) + , _d(NULL) + , _path(NULL) + , _isDirectory(false) + , _written(false) +{ + char * temp = (char *)malloc(strlen(fpath)+strlen(_fs->_mountpoint)+1); + if(!temp) { + return; + } + + strcpy(temp, _fs->_mountpoint); + strcat(temp, fpath); + + _path = strdup(fpath); + if(!_path) { + log_e("strdup(%s) failed", fpath); + free(temp); + return; + } + + if(!stat(temp, &_stat)) { + //file found + if (S_ISREG(_stat.st_mode)) { + _isDirectory = false; + _f = fopen(temp, mode); + if(!_f) { + log_e("fopen(%s) failed", temp); + } + if(_f && (_stat.st_blksize == 0)) + { + setvbuf(_f,NULL,_IOFBF,DEFAULT_FILE_BUFFER_SIZE); + } + } else if(S_ISDIR(_stat.st_mode)) { + _isDirectory = true; + _d = opendir(temp); + if(!_d) { + log_e("opendir(%s) failed", temp); + } + } else { + log_e("Unknown type 0x%08X for file %s", ((_stat.st_mode)&_IFMT), temp); + } + } else { + //file not found + if(!mode || mode[0] == 'r') { + //try to open as directory + _d = opendir(temp); + if(_d) { + _isDirectory = true; + } else { + _isDirectory = false; + //log_w("stat(%s) failed", temp); + } + } else { + //lets create this new file + _isDirectory = false; + _f = fopen(temp, mode); + if(!_f) { + log_e("fopen(%s) failed", temp); + } + if(_f && (_stat.st_blksize == 0)) + { + setvbuf(_f,NULL,_IOFBF,DEFAULT_FILE_BUFFER_SIZE); + } + } + } + free(temp); +} + +VFSFileImpl::~VFSFileImpl() +{ + close(); +} + +void VFSFileImpl::close() +{ + if(_path) { + free(_path); + _path = NULL; + } + if(_isDirectory && _d) { + closedir(_d); + _d = NULL; + _isDirectory = false; + } else if(_f) { + fclose(_f); + _f = NULL; + } +} + +VFSFileImpl::operator bool() +{ + return (_isDirectory && _d != NULL) || _f != NULL; +} + +time_t VFSFileImpl::getLastWrite() { + _getStat() ; + return _stat.st_mtime; +} + +void VFSFileImpl::_getStat() const +{ + if(!_path) { + return; + } + char * temp = (char *)malloc(strlen(_path)+strlen(_fs->_mountpoint)+1); + if(!temp) { + return; + } + + strcpy(temp, _fs->_mountpoint); + strcat(temp, _path); + + if(!stat(temp, &_stat)) { + _written = false; + } + free(temp); +} + +size_t VFSFileImpl::write(const uint8_t *buf, size_t size) +{ + if(_isDirectory || !_f || !buf || !size) { + return 0; + } + _written = true; + return fwrite(buf, 1, size, _f); +} + +size_t VFSFileImpl::read(uint8_t* buf, size_t size) +{ + if(_isDirectory || !_f || !buf || !size) { + return 0; + } + + return fread(buf, 1, size, _f); +} + +void VFSFileImpl::flush() +{ + if(_isDirectory || !_f) { + return; + } + fflush(_f); + // workaround for https://github.com/espressif/arduino-esp32/issues/1293 + fsync(fileno(_f)); +} + +bool VFSFileImpl::seek(uint32_t pos, SeekMode mode) +{ + if(_isDirectory || !_f) { + return false; + } + auto rc = fseek(_f, pos, mode); + return rc == 0; +} + +size_t VFSFileImpl::position() const +{ + if(_isDirectory || !_f) { + return 0; + } + return ftell(_f); +} + +size_t VFSFileImpl::size() const +{ + if(_isDirectory || !_f) { + return 0; + } + if (_written) { + _getStat(); + } + return _stat.st_size; +} + +/* +* Change size of files internal buffer used for read / write operations. +* Need to be called right after opening file before any other operation! +*/ +bool VFSFileImpl::setBufferSize(size_t size) +{ + if(_isDirectory || !_f) { + return 0; + } + int res = setvbuf(_f,NULL,_IOFBF,size); + return res == 0; +} + +const char* VFSFileImpl::path() const +{ + return (const char*) _path; +} + +const char* VFSFileImpl::name() const +{ + return pathToFileName(path()); +} + +//to implement +boolean VFSFileImpl::isDirectory(void) +{ + return _isDirectory; +} + +FileImplPtr VFSFileImpl::openNextFile(const char* mode) +{ + if(!_isDirectory || !_d) { + return FileImplPtr(); + } + struct dirent *file = readdir(_d); + if(file == NULL) { + return FileImplPtr(); + } + if(file->d_type != DT_REG && file->d_type != DT_DIR) { + return openNextFile(mode); + } + + size_t pathLen = strlen(_path); + size_t fileNameLen = strlen(file->d_name); + char * name = (char *)malloc(pathLen+fileNameLen+2); + + if(name == NULL) { + return FileImplPtr(); + } + + strcpy(name, _path); + + if ((file->d_name[0] != '/') && (_path[pathLen - 1] != '/')) + { + strcat(name, "/"); + } + + strcat(name, file->d_name); + + FileImplPtr fileImplPtr = std::make_shared(_fs, name, mode); + free(name); + return fileImplPtr; +} + +boolean VFSFileImpl::seekDir(long position){ + if(!_d){ + return false; + } + seekdir(_d, position); + return true; +} + + +String VFSFileImpl::getNextFileName() +{ + if (!_isDirectory || !_d) { + return ""; + } + struct dirent *file = readdir(_d); + if (file == NULL) { + return ""; + } + if (file->d_type != DT_REG && file->d_type != DT_DIR) { + return ""; + } + String fname = String(file->d_name); + String name = String(_path); + if (!fname.startsWith("/") && !name.endsWith("/")) { + name += "/"; + } + name += fname; + return name; +} + +String VFSFileImpl::getNextFileName(bool *isDir) +{ + if (!_isDirectory || !_d) { + return ""; + } + struct dirent *file = readdir(_d); + if (file == NULL) { + return ""; + } + if (file->d_type != DT_REG && file->d_type != DT_DIR) { + return ""; + } + String fname = String(file->d_name); + String name = String(_path); + if (!fname.startsWith("/") && !name.endsWith("/")) { + name += "/"; + } + name += fname; + + // check entry is a directory + if (isDir) { + *isDir = (file->d_type == DT_DIR); + } + return name; +} + +void VFSFileImpl::rewindDirectory(void) +{ + if(!_isDirectory || !_d) { + return; + } + rewinddir(_d); +} diff --git a/test/test_esp32_file_buffer_fix.py b/test/test_esp32_file_buffer_fix.py new file mode 100644 index 00000000..c5b53738 --- /dev/null +++ b/test/test_esp32_file_buffer_fix.py @@ -0,0 +1,302 @@ +#!/usr/bin/env python3 +"""Portable witnesses for the exact pinned Arduino/newlib buffer admission fix.""" +import hashlib +import importlib.util +from pathlib import Path +import subprocess +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[1] +FIXTURES = ROOT / "test/fixtures/esp32_file_buffer" +SPEC = importlib.util.spec_from_file_location("fs_buffer_fix", ROOT / "scripts/esp32_file_buffer_fix.py") +FIX = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(FIX) +RAW = (FIXTURES / "vfs_api.cpp").read_text() +PATCHED = FIX.patched_vfs_source(RAW) + + +def method(source, signature): + start = source.index(signature) + opening = source.index("{", start) + depth = 0 + for end in range(opening, len(source)): + depth += (source[end] == "{") - (source[end] == "}") + if depth == 0: + return source[start:end + 1] + raise AssertionError("unclosed function") + + +METHOD_PREFIX = r''' +#include +#include +#include +#include +#include +#include +struct MeshFILE { + struct { unsigned char* _base; int _size; } _bf{}; + int response = 0, selected_size = 0; +}; +static unsigned calls = 0, payload_reads = 0, payload_writes = 0; +#define FILE MeshFILE +#define _IOFBF 0 +static int setvbuf(FILE* f, char* supplied, int mode, size_t requested) { + assert(f && supplied == nullptr && mode == _IOFBF); + ++calls; + f->_bf._size = f->selected_size ? f->selected_size : (requested ? requested : 4096); + return f->response; +} +struct VFSFileImpl { + bool _isDirectory; + FILE* _f; + bool setBufferSize(size_t); +}; +''' + +METHOD_MAIN = r''' +int main() { + FILE stream{}; + VFSFileImpl file{false, &stream}; + for (size_t size : {1u, 64u, 128u, 152u, 251u, 4096u}) { + stream.selected_size = 0; + assert(file.setBufferSize(size)); + assert(static_cast(stream._bf._size) == size); + } + stream.response = -1; + stream.selected_size = 251; + assert(!file.setBufferSize(251)); // even an equal stored size cannot mask error + stream.response = 0; + stream.selected_size = 4096; + assert(!file.setBufferSize(251)); // real newlib successful larger fallback + assert(!file.setBufferSize(128)); + stream.selected_size = -1; + assert(!file.setBufferSize(251)); + stream.selected_size = 152; + assert(!file.setBufferSize(251)); // any mismatched positive size is rejected + stream.selected_size = 4096; + assert(file.setBufferSize(0)); // unchanged choose-default public semantics + stream.response = -1; + assert(!file.setBufferSize(0)); + const unsigned before = calls; + VFSFileImpl directory{true, &stream}, missing{false, nullptr}; + assert(!directory.setBufferSize(251) && !missing.setBufferSize(251)); + assert(calls == before && payload_reads == 0 && payload_writes == 0); +} +''' + +TRANSACTION_MOCK = r''' +class Filesystem; +class File { + Filesystem* _fs = nullptr; + bool _writer = false; + std::shared_ptr _stream; +public: + File() = default; + File(Filesystem*, bool); + operator bool() const { return static_cast(_stream); } + bool setBufferSize(size_t size) { + VFSFileImpl impl{false, _stream.get()}; + return impl.setBufferSize(size); + } + size_t write(const uint8_t*, size_t); + size_t read(uint8_t*, size_t) { ++payload_reads; return 0; } + size_t size() const; + void flush() {} + void close() { _stream.reset(); } +}; +class Filesystem { +public: + bool writer_fallback = false, reader_fallback = false; + unsigned renames = 0; + size_t length = 0; + std::set paths{"/contacts3"}; + bool exists(const char* path) { return paths.count(path); } + bool remove(const char* path) { return paths.erase(path); } + bool rename(const char*, const char*) { ++renames; return true; } + File open(const char* path, const char* mode, bool = false) { + if (*mode == 'w') { paths.insert(path); length = 0; } + return File(this, *mode == 'w'); + } +}; +File::File(Filesystem* fs, bool writer) : _fs(fs), _writer(writer), _stream(new FILE{}) { + _stream->selected_size = (writer ? fs->writer_fallback : fs->reader_fallback) ? 4096 : 0; +} +size_t File::size() const { return _fs->length; } +size_t File::write(const uint8_t*, size_t len) { ++payload_writes; _fs->length += len; return len; } +#define FILESYSTEM Filesystem +#define ESP32_PLATFORM 1 +''' + +TRANSACTION_MAIN = r''' +int main() { + uint8_t record[mesh::storage::CONTACT_RECORD_SIZE]{}; + Filesystem fs; + fs.writer_fallback = true; + { + mesh::ContactFileTransaction transaction(&fs, "/contacts3"); + assert(!transaction); + assert(transaction.write(record, sizeof(record)) == 0); + assert(!transaction.commit()); + assert(payload_writes == 0 && payload_reads == 0 && fs.renames == 0); + } + assert(fs.exists("/contacts3") && !fs.exists("/contacts3.tmp")); + fs.writer_fallback = false; + fs.reader_fallback = true; + { + mesh::ContactFileTransaction transaction(&fs, "/contacts3"); + assert(transaction); + assert(transaction.write(record, sizeof(record)) == sizeof(record)); + assert(transaction.serviceCommit() == mesh::ContactFileTransaction::CommitProgress::Pending); + assert(transaction.serviceCommit() == mesh::ContactFileTransaction::CommitProgress::Failed); + assert(payload_reads == 0 && payload_writes == 1 && fs.renames == 0); + } + assert(fs.exists("/contacts3") && !fs.exists("/contacts3.tmp")); +} +''' + + +class FileBufferFixTest(unittest.TestCase): + def compile_run(self, source, *, success=True, compile_failure=False, headers=None): + with tempfile.TemporaryDirectory(prefix="mesh-fs-buffer-") as directory: + temp = Path(directory) + (temp / "test.cpp").write_text(source) + for name, contents in (headers or {}).items(): + (temp / name).write_text(contents) + built = subprocess.run([ + "c++", "-std=c++11", "-Os", "-Wall", "-Wextra", "-Werror", + "-Wno-unused-parameter", "-fsanitize=address,undefined", + "-fno-sanitize-recover=all", "-fno-pie", "-no-pie", + "-I", str(temp), "-I", str(ROOT / "src"), + str(temp / "test.cpp"), "-o", str(temp / "test")], + capture_output=True, text=True, timeout=30) + if compile_failure: + self.assertNotEqual(built.returncode, 0) + self.assertIn("MeshCore FS buffer fix", built.stderr) + return + self.assertEqual(built.returncode, 0, built.stdout + built.stderr) + result = subprocess.run([str(temp / "test")], capture_output=True, text=True, timeout=10) + if success: + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + else: + self.assertNotEqual(result.returncode, 0, "unsafe negative control unexpectedly passed") + self.assertIn("file.setBufferSize(251)", result.stderr) + + def test_actual_method_honored_fallback_errors_zero_and_no_io(self): + self.compile_run(METHOD_PREFIX + method(PATCHED, "bool VFSFileImpl::setBufferSize(") + METHOD_MAIN) + + def test_upstream_success_only_negative_control_rejects_fallback(self): + self.compile_run(METHOD_PREFIX + method(RAW, "bool VFSFileImpl::setBufferSize(") + METHOD_MAIN, + success=False) + + def test_actual_transaction_rejects_writer_and_verifier_fallback_before_io(self): + transaction = (ROOT / "src/helpers/ContactFileTransaction.h").read_text().replace( + '#include "IdentityStore.h"', '#include "identity_seam.h"').replace( + '#include "PersistentStoreFormat.h"', '#include ').replace( + '#pragma once\n', '', 1) + self.compile_run(METHOD_PREFIX + method(PATCHED, "bool VFSFileImpl::setBufferSize(") + + TRANSACTION_MOCK + transaction + TRANSACTION_MAIN, + headers={"identity_seam.h": "#pragma once\n"}) + + def layout_source(self): + # Execute the exact header's reviewed prefix with 32-bit pointer values + # on a native 64-bit host. The target copy additionally asserts void*32. + reent = (FIXTURES / "newlib_sys_reent.h").read_text() + sbuf = method(reent, "struct __sbuf {") + ";\n" + start = reent.index("struct __sFILE {") + end = reent.index("#ifdef _REENT_SMALL", start) + prefix = reent[start:end] + "};\n" + fields = (sbuf + prefix).replace("unsigned char *", "ModelPointer ") + source = "#include \nusing ModelPointer = uint32_t;\n" + fields + source += "using FILE = __sFILE;\n" + source += FIX.LAYOUT_GUARD.replace("sizeof(void*)", "sizeof(ModelPointer)") + source += "int main() {}\n" + return source + + def test_exact_pinned_file_prefix_and_newlib_guards(self): + version = (FIXTURES / "_newlib_version.h").read_text() + headers = {"newlib.h": version} + self.compile_run(self.layout_source(), headers=headers) + for broken in (self.layout_source().replace("struct __sbuf _bf;", "int added; struct __sbuf _bf;"), + self.layout_source().replace("int\t_size;", "long long\t_size;")): + self.compile_run(broken, headers=headers, compile_failure=True) + self.compile_run(self.layout_source(), headers={"newlib.h": version.replace( + "#define __NEWLIB__ 3", "#define __NEWLIB__ 4")}, compile_failure=True) + + def test_exact_fingerprints_shape_normalization_and_idempotence(self): + self.assertEqual(hashlib.sha256((FIXTURES / "vfs_api.cpp").read_bytes()).hexdigest(), FIX.PINNED_VFS_SHA256) + self.assertEqual(FIX.patched_vfs_source(RAW.replace("\n", "\r\n")), PATCHED) + self.assertEqual(FIX.patched_vfs_source(PATCHED), PATCHED) + self.assertEqual(PATCHED, RAW.replace(FIX.INCLUDE, FIX.INCLUDE + FIX.LAYOUT_GUARD, 1).replace( + FIX.ORIGINAL_METHOD, FIX.STRICT_METHOD, 1)) + for changed in (RAW + "\n", PATCHED + "\n", RAW.replace("return res == 0;", "return true;", 1)): + with self.assertRaises(RuntimeError): + FIX.patched_vfs_source(changed) + expected = { + "esp_arduino_version.h": "b63e57731a24289b06eb96ded9e481ea719f4cde2ce7c41bd83c7646e0a6fe51", + "newlib_sys_reent.h": "33391d61a83ebbfb122e80514fbf7ed4d770556d8e8d46e029addbbf6077cd79", + "_newlib_version.h": "93a397574a25b9c143cee7fcb86d8fc9a0c9ba418e4710a30d2d19ba1e4120db"} + for name, digest in expected.items(): + self.assertEqual(hashlib.sha256((FIXTURES / name).read_bytes()).hexdigest(), digest) + + def test_build_local_copy_all_usb_modes_guards_and_shared_bytes(self): + class Node: + def __init__(self, path): self.path = path + def srcnode(self): return self + def get_abspath(self): return str(self.path) + class Env(dict): + def __init__(self, defines, build): + super().__init__(CPPDEFINES=defines) + self.build, self.paths, self.middleware = build, [], [] + def subst(self, value): + assert value == "$BUILD_DIR" + return str(self.build) + def File(self, path): return path + def AppendUnique(self, **kwargs): + for path in kwargs["CPPPATH"]: + if path not in self.paths: self.paths.append(path) + def AddBuildMiddleware(self, callback, pattern): self.middleware.append((callback, pattern)) + with tempfile.TemporaryDirectory(prefix="mesh-fs-copy-") as directory: + temp = Path(directory) + source = temp / "sdk/libraries/FS/src/vfs_api.cpp" + source.parent.mkdir(parents=True) + source.write_bytes((FIXTURES / "vfs_api.cpp").read_bytes()) + version = temp / "sdk/cores/esp32/esp_arduino_version.h" + version.parent.mkdir(parents=True) + version.write_bytes((FIXTURES / "esp_arduino_version.h").read_bytes()) + node = Node(source) + for defines in ([], ["RP2040_PLATFORM"], [("ESP32_PLATFORM", 0)]): + env = Env(defines, temp / "untouched") + self.assertIs(FIX.replace_vfs_source(env, node), node) + self.assertFalse(env.build.exists()) + for index, usb_mode in enumerate((0, 1, None)): + defines = [("ESP32_PLATFORM", 1)] + if usb_mode is not None: + defines += [("ARDUINO_USB_MODE", usb_mode), ("ARDUINO_USB_CDC_ON_BOOT", 1)] + env = Env(defines, temp / ("build" + str(index))) + result = Path(FIX.replace_vfs_source(env, node)) + self.assertEqual(result.read_text(), PATCHED) + self.assertEqual(env.paths, [str(source.parent)]) + self.assertEqual(source.read_bytes(), (FIXTURES / "vfs_api.cpp").read_bytes()) + before = result.stat().st_mtime_ns + self.assertEqual(Path(FIX.replace_vfs_source(env, node)).stat().st_mtime_ns, before) + env = Env(["ESP32"], temp / "strict") + unrelated = Node(temp / "unrelated.cpp") + self.assertIs(FIX.replace_vfs_source(env, unrelated), unrelated) + FIX.install(env) + self.assertEqual(env.middleware, [(FIX.replace_vfs_source, "*libraries*FS*src*vfs_api.cpp")]) + for header in ("", version.read_text().replace("VERSION_PATCH 17", "VERSION_PATCH 18"), + version.read_text() + "\n#define ESP_ARDUINO_VERSION_MAJOR 2\n"): + version.write_text(header) + with self.assertRaises(RuntimeError): FIX.replace_vfs_source(env, node) + self.assertFalse(env.build.exists()) + version.unlink() + with self.assertRaises(RuntimeError): FIX.replace_vfs_source(env, node) + version.write_bytes((FIXTURES / "esp_arduino_version.h").read_bytes()) + source.write_text(RAW + "\n") + with self.assertRaises(RuntimeError): FIX.replace_vfs_source(env, node) + self.assertFalse(env.build.exists()) + + +if __name__ == "__main__": + unittest.main()