diff --git a/.github/size_check.py b/.github/size_check.py new file mode 100644 index 0000000..a10d402 --- /dev/null +++ b/.github/size_check.py @@ -0,0 +1,52 @@ +"""Fail the build when the firmware leaves too little room in its app slot. + +PlatformIO only fails once the firmware no longer fits. Releases are built at +CORE_DEBUG_LEVEL=2 because the ESP32 image barely fits its 1.5 MB OTA slot, so +this warns while there is still room to act. + +Usage: python .github/size_check.py [min free KB] +Writes a line to $GITHUB_STEP_SUMMARY when it is set. +""" +import os +import struct +import sys + +ENTRY = struct.Struct(" 3 else 8 * 1024 + used = os.path.getsize(os.path.join(build_dir, "firmware.bin")) + slot = app_slot_size(os.path.join(build_dir, "partitions.bin")) + free = slot - used + ok = free >= min_free + line = (f"| {name} | {used:,} | {slot:,} | {free / 1024:.1f} KB | " + f"{'OK' if ok else f'FAIL: under {min_free // 1024} KB free'} |") + print(line) + summary = os.environ.get("GITHUB_STEP_SUMMARY") + if summary: + with open(summary, "a", encoding="utf-8") as f: + f.write("| Env | Firmware | App slot | Free | |\n|---|---|---|---|---|\n") + f.write(line + "\n") + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index bbc23e1..b94857f 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -21,8 +21,9 @@ jobs: with: python-version: "3.12" + # pioarduino's fork of PlatformIO Core, which the pioarduino platform needs. - name: Install PlatformIO - run: pip install platformio + run: pip install pioarduino - name: Run native unit tests run: pio test -e native @@ -40,7 +41,9 @@ jobs: # fwname: firmware-only binary for OTA updates # obname: onboard image (bootloader + partitions + firmware + littlefs merged) # for flashing a new chip via USB/serial - # addr_bootloader: chip-dependent (ESP32: 0x1000, C3/S3/C6: 0x0) + # addr_bootloader: chip-dependent (ESP32: 0x1000, S3/C6: 0x0) + # No C3: on ESP-IDF 5.5 its image no longer fits the 1.5 MB OTA slot + # (docs/adr/0003-pioarduino-idf55.md). C3 boards stay on 3.0.x. # addr_fs: must match spiffs/littlefs offset in esp32_3MB.csv include: - env: esp32dev @@ -51,14 +54,6 @@ jobs: fwname: SomfyController.ino.esp32.bin fsname: SomfyController.littlefs.esp32.bin obname: SomfyController.onboard.esp32.bin - - env: esp32c3 - name: ESP32-C3 - chip: ESP32-C3 - addr_bootloader: "0x0" - addr_fs: "0x310000" - fwname: SomfyController.ino.esp32c3.bin - fsname: SomfyController.littlefs.esp32c3.bin - obname: SomfyController.onboard.esp32c3.bin - env: esp32s3 name: ESP32-S3 chip: ESP32-S3 @@ -112,7 +107,7 @@ jobs: python-version: "3.12" - name: Install PlatformIO and esptool - run: pip install platformio esptool + run: pip install pioarduino esptool - name: Build LittleFS image run: pio run -e ${{ matrix.env }} -t buildfs @@ -123,6 +118,9 @@ jobs: - name: Build firmware run: pio run -e ${{ matrix.env }} + - name: Check firmware size + run: python .github/size_check.py .pio/build/${{ matrix.env }} ${{ matrix.env }} 8 + - name: Restore LittleFS image run: cp littlefs.bin .pio/build/${{ matrix.env }}/littlefs.bin diff --git a/AGENTS.md b/AGENTS.md index b902d5b..c4089ab 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,3 +2,7 @@ - Fix any test or type errors until the whole suite is green. - Add or update tests for the code you change, even if nobody asked. - to test run `pio test -e native` + +## Building +- Firmware builds need pioarduino's Core (`pip install pioarduino`, or the pioarduino IDE extension). Stock PlatformIO Core refuses the pinned platform, see [ADR 0003](docs/adr/0003-pioarduino-idf55.md). +- The platform installs its Python packages into `~/.platformio/penv` and fails on Windows while `pio.exe` from that env is running. Run builds as `python -m platformio` from that env's Python. diff --git a/README.md b/README.md index 19c5a85..47f6043 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,9 @@ * Backup does no longer use LITTLEFS. Only RAM is used. Storage should not wear out so quick. * Added uptime info in web UI +## Building +The firmware builds on [pioarduino](https://github.com/pioarduino/platform-espressif32) (Arduino 3.x on ESP-IDF 5.5), not the stock PlatformIO platform. In VS Code use the **pioarduino IDE** extension instead of PlatformIO IDE. On the command line, `pip install pioarduino` gives you `pio`. Run the unit tests with `pio test -e native`. [ADR 0003](docs/adr/0003-pioarduino-idf55.md) explains why. + ## There is a partition layout modification it's just better to perform full ESP32 wipeout. Perform backup then wipeout the firmware and restore from the bkp. * fwname: firmware-only binary for OTA updates * obname: onboard image (bootloader + partitions + firmware + littlefs merged) for flashing a new chip via USB/serial diff --git a/docs/SECURITY_PLAN.md b/docs/SECURITY_PLAN.md index 970c729..cd5576b 100644 --- a/docs/SECURITY_PLAN.md +++ b/docs/SECURITY_PLAN.md @@ -107,5 +107,6 @@ Everything else is checked with the `esp32devdbg` build, plus curl and websocket ## Not in this plan - TLS. +- Checking GitHub's certificate in `GitOTA`. It calls `setInsecure()`, so anyone who can intercept the update download can serve their own firmware. ESP-IDF 5.5 ([ADR 0003](adr/0003-pioarduino-idf55.md)) ships a certificate bundle, so this is possible now. It needs its own change: the full bundle costs more flash than the ESP32 build has spare, and pinning GitHub's root certificates might fit. - Session tokens with expiry. - Contents of the binary backup file (it holds no passwords, but it does hold rolling codes; `/backup` becomes a config route). diff --git a/docs/adr/0003-pioarduino-idf55.md b/docs/adr/0003-pioarduino-idf55.md new file mode 100644 index 0000000..3a6baf0 --- /dev/null +++ b/docs/adr/0003-pioarduino-idf55.md @@ -0,0 +1,18 @@ +# Firmware builds on pioarduino (Arduino 3.x, ESP-IDF 5.5), pinned to one release + +We moved every firmware env from `espressif32 @ 6.13.0` (Arduino 2.0.17 on ESP-IDF 4.4.7) to pioarduino's `55.03.312` (Arduino 3.3.12 on ESP-IDF 5.5). IDF 4.4 no longer gets security fixes for Wi-Fi, lwIP and mbedTLS, and the planned RMT radio needs IDF 5's RMT driver. The official PlatformIO platform never shipped Arduino 3.x, so pioarduino, a community fork, is the only way to get it in PlatformIO. We pin one release URL rather than `stable`, so a framework upgrade is always its own PR with its own size check and hardware test. + +## Considered Options + +- **Stay on IDF 4.4.** Rejected: no more security fixes, and no new RMT driver. +- **Rebuild the IDF libraries with `custom_sdkconfig`** to leave out PPP, NAT and provisioning (about 20 KB the firmware never uses). Deferred: it makes builds much slower, and log level 2 already makes the ESP32 build fit. +- **Grow the app slots in the partition table.** Rejected: an OTA update can't change the partition table, so every existing device would need a USB flash. + +## Consequences + +- **Firmware size.** The ESP32 image grows from 1,382,640 bytes (v3.0.14) to 1,555,472 at `CORE_DEBUG_LEVEL=2`, leaving 17 KB of the 1.5 MB OTA slot. At level 3 it doesn't fit, so release envs build at level 2 (warnings and errors) and debug envs stay at 3. PlatformIO's "Flash:" figure counts ELF sections, not the image, and reads about 45 KB low, so CI checks `firmware.bin` against the app partition and fails under 8 KB free (`.github/size_check.py`). +- **No more C3 builds.** Its image grows by 214 KB to 1,651,216 bytes, 76.5 KB over the OTA slot even at level 2. C3 boards stay on the 3.0.x releases. +- **Ethernet PHY ids are ours.** Arduino 3.x renumbered `eth_phy_type_t` (`ETH_PHY_GENERIC` is 0). NVS, backups and the web UI keep the Arduino 2.x numbers as `eth_phy_id_t` (`src/EthPhy.h`), translated by name when Ethernet starts. Never cast a stored id to the framework enum. +- **Ethernet only on the original ESP32.** Arduino 3.x has the RMII MAC API only where `CONFIG_ETH_USE_ESP32_EMAC` is set. S3 and C6 build without Ethernet. +- **Updates roll back until proven.** We override the core's `verifyRollbackLater()` hook to return true, and `loop()` marks the new firmware valid once it has run 60 s and the network is up. A reset before then boots the previous firmware. The LittleFS image an update wrote stays, so the old firmware runs with the new web UI. +- **Tooling.** pioarduino needs its own Core fork (`pip install pioarduino`, the pioarduino IDE extension in VS Code), which it installs into `~/.platformio/penv` in place of PlatformIO Core. diff --git a/platformio.ini b/platformio.ini index 12645b4..6ddfc27 100644 --- a/platformio.ini +++ b/platformio.ini @@ -15,7 +15,9 @@ src_dir = src ; Shared by every firmware env (they extend it). Kept out of [env] so the ; native test env doesn't inherit the Arduino framework and ESP32 libraries. [esp32] -platform = espressif32 @ 6.13.0 +; pioarduino: Arduino 3.3.12 on ESP-IDF 5.5. Pinned to a release so an +; upgrade is its own PR (docs/adr/0003-pioarduino-idf55.md). +platform = https://github.com/pioarduino/platform-espressif32/releases/download/55.03.312/platform-espressif32.zip framework = arduino ; Unit tests only run on the PC: pio test -e native test_ignore = * @@ -30,8 +32,9 @@ extra_scripts = pre:minify.py post:archive_elf.py board_build.filesystem = littlefs -build_flags = - -DCORE_DEBUG_LEVEL=3 +; Each env adds a CORE_DEBUG_LEVEL: 2 (warnings) for release builds, which +; only fit the 1.5 MB OTA slot without the info logs, 3 for debug builds. +build_flags = -DCONFIG_ESP_COREDUMP_ENABLE_TO_FLASH=1 -DCONFIG_ESP_COREDUMP_DATA_FORMAT_ELF=1 -DCONFIG_ESP_COREDUMP_CHECKSUM_CRC32=1 @@ -57,7 +60,9 @@ build_flags = extends = esp32 board = esp32dev board_build.partitions = esp32_3MB.csv -build_flags = ${dualcore.build_flags} +build_flags = + ${dualcore.build_flags} + -DCORE_DEBUG_LEVEL=2 [env:esp32devdbg] extends = esp32 @@ -73,39 +78,26 @@ board_build.partitions = esp32_huge_app.csv ; KBs of flash per file, so treat every file as large. build_flags = ${dualcore.build_flags} + -DCORE_DEBUG_LEVEL=3 --param=large-unit-insns=1000 -[env:esp32c3] -extends = esp32 -board = esp32-c3-devkitm-1 -board_build.partitions = esp32_3MB.csv - [env:esp32s3] extends = esp32 board = esp32-s3-devkitc-1 -build_flags = ${dualcore.build_flags} +build_flags = + ${dualcore.build_flags} + -DCORE_DEBUG_LEVEL=2 +; Released as a debug build with one 3 MB app slot, so C6 boards update over +; USB, not OTA. [env:esp32c6] extends = esp32 -platform = https://github.com/pioarduino/platform-espressif32/releases/download/stable/platform-espressif32.zip board = seeed_xiao_esp32c6 build_type = debug board_build.partitions = esp32_huge_app.csv build_flags = ${esp32.build_flags} - -DCONFIG_FREERTOS_HZ=1000 - -include "driver/gpio.h" -lib_deps = - ; Low-Level libs, frameowrks and drivers - ESP32Async/AsyncTCP - https://github.com/Viproz/SmartRC-CC1101-Driver-Lib.git - ; Commmunication Protocol - links2004/WebSockets - https://github.com/hmueller01/pubsubclient3.git - ; Data-Parsing - bblanchon/ArduinoJson - ; High-level wrappers/servers - ESP32Async/ESPAsyncWebServer + -DCORE_DEBUG_LEVEL=3 ; Host-side unit tests: pio test -e native (needs gcc/g++ on PATH). ; Only the sources the tests cover are built; test/stubs stands in for Arduino.h. diff --git a/src/ESPNetwork.cpp b/src/ESPNetwork.cpp index 5125a9c..6a51e58 100644 --- a/src/ESPNetwork.cpp +++ b/src/ESPNetwork.cpp @@ -396,7 +396,7 @@ bool ESPNetwork::connectWired() { if(!ethPhyType(settings.Ethernet.phyType, phyType)) ESP_LOGE(TAG, "Ethernet PHY type %u is not supported", (unsigned int)settings.Ethernet.phyType); else - begun = ETH.begin(settings.Ethernet.phyAddress, settings.Ethernet.PWRPin, settings.Ethernet.MDCPin, settings.Ethernet.MDIOPin, phyType, settings.Ethernet.CLKMode); + begun = ETH.begin(phyType, settings.Ethernet.phyAddress, settings.Ethernet.MDCPin, settings.Ethernet.MDIOPin, settings.Ethernet.PWRPin, settings.Ethernet.CLKMode); if(!begun) { ESP_LOGE(TAG, "Ethernet Begin failed"); this->ethStarted = false; diff --git a/src/LogLock.cpp b/src/LogLock.cpp index 8c68d63..0d131f6 100644 --- a/src/LogLock.cpp +++ b/src/LogLock.cpp @@ -18,7 +18,6 @@ // progress (see LogLockPolicy.h). #include #include -#include "esp_idf_version.h" #include "freertos/FreeRTOS.h" #include "freertos/semphr.h" #include "freertos/task.h" @@ -41,19 +40,12 @@ static volatile TaskHandle_t logOwner = nullptr; // line takes about 20ms at 115200 baud. #define LOG_LOCK_WAIT_MS 20 -#if ESP_IDF_VERSION_MAJOR >= 5 -#define taskOnCore(core) xTaskGetCurrentTaskHandleForCore(core) -#define taskCore(task) xTaskGetCoreID(task) -#else -#define taskOnCore(core) xTaskGetCurrentTaskHandleForCPU(core) -#define taskCore(task) xTaskGetAffinity(task) -#endif static LogHolder logHolder(TaskHandle_t owner) { if(!owner) return LogHolder::unknown; for(BaseType_t core = 0; core < portNUM_PROCESSORS; core++) - if(taskOnCore(core) == owner) return LogHolder::running; + if(xTaskGetCurrentTaskHandleForCore(core) == owner) return LogHolder::running; // A task pinned elsewhere only runs once whatever preempted it there is done. - const BaseType_t core = taskCore(owner); + const BaseType_t core = xTaskGetCoreID(owner); if(core == tskNO_AFFINITY || core == xPortGetCoreID()) return LogHolder::preemptedHere; return LogHolder::preemptedElsewhere; } diff --git a/src/SomfyController.ino b/src/SomfyController.ino index fb1dbc2..0d7d973 100644 --- a/src/SomfyController.ino +++ b/src/SomfyController.ino @@ -16,18 +16,20 @@ #include "GitOTA.h" #include "esp_core_dump.h" #include "esp_system.h" -#include "esp_idf_version.h" -#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 0, 0) #include "esp_app_desc.h" -#define getRunningElfSha esp_app_get_elf_sha256 -#else #include "esp_ota_ops.h" -#define getRunningElfSha esp_ota_get_app_elf_sha256 -#endif #include "Timing.h" static const char *TAG = "Main"; #define LOOP_SLOW_MS 100 +// How long an updated firmware has to run, and then have the network up, before +// it is kept. A reset before then rolls back to the previous firmware. +#define OTA_PROVEN_MS 60000 + +// Overrides the core's weak hook so it leaves an updated firmware unverified +// at boot. loop() confirms it once otaProven(). +extern "C" bool verifyRollbackLater() { return true; } +static bool otaPending = false; ConfigSettings settings; Web webServer; @@ -101,7 +103,7 @@ void setup() { static_assert(sizeof(crashedSha) <= sizeof(summary.app_elf_sha256), "copy would read past app_elf_sha256"); SETCHARPROP(crashedSha, (const char *)summary.app_elf_sha256, sizeof(crashedSha)); char runningSha[APP_ELF_SHA256_SZ] = {0}; - getRunningElfSha(runningSha, sizeof(runningSha)); + esp_app_get_elf_sha256(runningSha, sizeof(runningSha)); if(strcmp(crashedSha, runningSha) == 0) CRASH_LOG(" Build: %s (running %s, same build)", crashedSha, runningSha); else @@ -129,14 +131,15 @@ void setup() { net.setup(); somfy.begin(); //git.checkForUpdate(); -#if ESP_ARDUINO_VERSION_MAJOR >= 3 + // The core starts the watchdog with a 5s timeout before setup(), so + // esp_task_wdt_init() would fail and keep that. const esp_task_wdt_config_t wdt_config = { .timeout_ms = 15000, .idle_core_mask = 1, .trigger_panic = true }; - esp_task_wdt_init(&wdt_config); -#else - esp_task_wdt_init(15, true); //enable panic so ESP32 restarts -#endif + esp_task_wdt_reconfigure(&wdt_config); esp_task_wdt_add(NULL); //add current thread to WDT watch - + esp_ota_img_states_t otaState; + otaPending = esp_ota_get_state_partition(esp_ota_get_running_partition(), &otaState) == ESP_OK && + otaState == ESP_OTA_IMG_PENDING_VERIFY; + if(otaPending) ESP_LOGW(TAG, "Updated firmware, keeping it once it has run %ds with the network up", OTA_PROVEN_MS / 1000); } void loop() { @@ -183,6 +186,11 @@ void loop() { net.end(); ESP.restart(); } + if(otaPending && otaProven(net.connected(), millis(), OTA_PROVEN_MS)) { + otaPending = false; + if(esp_ota_mark_app_valid_cancel_rollback() == ESP_OK) ESP_LOGI(TAG, "Updated firmware confirmed, rollback cancelled"); + else ESP_LOGE(TAG, "Unable to confirm the updated firmware"); + } esp_task_wdt_reset(); loopStats.record(pass.us()); } diff --git a/src/Utils.cpp b/src/Utils.cpp index f329951..75c77f0 100644 --- a/src/Utils.cpp +++ b/src/Utils.cpp @@ -147,6 +147,9 @@ void RssiBroadcast::sent(bool broadcast, int rssi, int channel, uint32_t now) { bool worthRoaming(bool connected, int currentRssi, int apRssi, int minGain) { return !connected || apRssi >= currentRssi + minGain; } +bool otaProven(bool networkUp, unsigned long uptimeMs, unsigned long minUptimeMs) { + return networkUp && uptimeMs >= minUptimeMs; +} const char *wifiAuthModeName(uint8_t authMode) { switch(authMode) { case 0: return "OPEN"; diff --git a/src/Utils.h b/src/Utils.h index e182535..27c9ba9 100644 --- a/src/Utils.h +++ b/src/Utils.h @@ -82,6 +82,10 @@ enum class heap_emit_t : uint8_t { none, client, broadcast, room }; // num is the socket client asked for, 255 for everyone. A broadcast goes out // every intervalMs; in between only the clients in room 0 get one. heap_emit_t heapEmitTarget(uint8_t num, uint32_t sinceLastMs, uint32_t intervalMs, bool roomHasClients); +// Whether a firmware the bootloader is holding for verification after an +// update has proven itself: it has run for minUptimeMs and the network is +// up. Until then a reset rolls back to the previous firmware. +bool otaProven(bool networkUp, unsigned long uptimeMs, unsigned long minUptimeMs); // How well loop() keeps up: the passes it made per second and its longest // pass, over the window since the last memStatus broadcast. Only the loop // task calls record(), rate(), maxUs() and publish(). publish() closes the diff --git a/test/test_utils/test_utils.cpp b/test/test_utils/test_utils.cpp index f3c2c2c..b17d0e0 100644 --- a/test/test_utils/test_utils.cpp +++ b/test/test_utils/test_utils.cpp @@ -257,6 +257,21 @@ static void test_worthRoaming_any_ap_while_disconnected() { TEST_ASSERT_TRUE(worthRoaming(false, 0, -90, 6)); } +/********************************************************************* + * otaProven + ********************************************************************/ +static void test_otaProven_after_the_minimum_uptime_with_network() { + TEST_ASSERT_TRUE(otaProven(true, 60000, 60000)); + TEST_ASSERT_TRUE(otaProven(true, 3600000, 60000)); +} +static void test_otaProven_not_before_the_minimum_uptime() { + TEST_ASSERT_FALSE(otaProven(true, 59999, 60000)); +} +static void test_otaProven_not_without_network() { + // A firmware that can't connect would leave the device unreachable. + TEST_ASSERT_FALSE(otaProven(false, 3600000, 60000)); +} + /********************************************************************* * wifiAuthModeName ********************************************************************/ @@ -384,6 +399,9 @@ int main(int argc, char **argv) { RUN_TEST(test_worthRoaming_needs_the_full_gain); RUN_TEST(test_worthRoaming_never_to_a_weaker_ap); RUN_TEST(test_worthRoaming_any_ap_while_disconnected); + RUN_TEST(test_otaProven_after_the_minimum_uptime_with_network); + RUN_TEST(test_otaProven_not_before_the_minimum_uptime); + RUN_TEST(test_otaProven_not_without_network); RUN_TEST(test_wifiAuthModeName_matches_the_idf_enum); RUN_TEST(test_wifiAuthModeName_unknown_past_the_shared_values); RUN_TEST(test_heapEmitTarget_one_client_always_gets_it);