build: move to pioarduino 55.03.312 (Arduino 3.3.12, ESP-IDF 5.5)

IDF 4.4 gets no more security fixes, and the planned RMT radio needs
IDF 5's RMT driver.  See docs/adr/0003-pioarduino-idf55.md.

- Every env on one pinned pioarduino release.  C6 drops its own
  library list and flags but stays a debug build on huge_app.
- Release envs build at CORE_DEBUG_LEVEL=2 so the ESP32 image fits its
  1.5 MB OTA slot (17 KB free); debug envs stay at 3.
- Drop the C3: its image is 76.5 KB over the OTA slot on IDF 5.5.
- CI installs pioarduino's Core and checks firmware.bin against the app
  partition, failing under 8 KB free.  PlatformIO's "Flash:" figure
  reads ~45 KB low and passed a C3 image that doesn't fit.
- esp_task_wdt_reconfigure(): on 3.x the core already starts the watchdog
  and esp_task_wdt_init() fails, so the C6 build kept the 5 s default.
- An updated firmware is kept only once it has run 60 s with the
  network up; a reset before then rolls back to the previous one.
- ETH.begin() takes the 3.x argument order; Arduino 2.x branches gone.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.github/size_check.py

@@ -0,0 +1,52 @@
+"""Fail the build when the firmware leaves too little room in its app slot.
+
+PlatformIO only fails once the firmware no longer fits.  Releases are built at
+CORE_DEBUG_LEVEL=2 because the ESP32 image barely fits its 1.5 MB OTA slot, so
+this warns while there is still room to act.
+
+Usage: python .github/size_check.py <build dir> <name> [min free KB]
+Writes a line to $GITHUB_STEP_SUMMARY when it is set.
+"""
+import os
+import struct
+import sys
+
+ENTRY = struct.Struct("<HBBII16sI")  # magic, type, subtype, offset, size, label, flags
+ENTRY_MAGIC = 0x50AA
+APP_TYPE = 0x00
+
+
+def app_slot_size(partitions_bin):
+    """The largest app partition in a built partition table."""
+    with open(partitions_bin, "rb") as f:
+        table = f.read()
+    largest = 0
+    for pos in range(0, len(table) - ENTRY.size + 1, ENTRY.size):
+        magic, ptype, _, _, size, _, _ = ENTRY.unpack_from(table, pos)
+        if magic != ENTRY_MAGIC:
+            break
+        if ptype == APP_TYPE:
+            largest = max(largest, size)
+    return largest
+
+
+def main():
+    build_dir, name = sys.argv[1], sys.argv[2]
+    min_free = int(sys.argv[3]) * 1024 if len(sys.argv) > 3 else 8 * 1024
+    used = os.path.getsize(os.path.join(build_dir, "firmware.bin"))
+    slot = app_slot_size(os.path.join(build_dir, "partitions.bin"))
+    free = slot - used
+    ok = free >= min_free
+    line = (f"| {name} | {used:,} | {slot:,} | {free / 1024:.1f} KB | "
+            f"{'OK' if ok else f'FAIL: under {min_free // 1024} KB free'} |")
+    print(line)
+    summary = os.environ.get("GITHUB_STEP_SUMMARY")
+    if summary:
+        with open(summary, "a", encoding="utf-8") as f:
+            f.write("| Env | Firmware | App slot | Free | |\n|---|---|---|---|---|\n")
+            f.write(line + "\n")
+    return 0 if ok else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/workflows/build.yaml

@@ -21,8 +21,9 @@ jobs:
         with:
           python-version: "3.12"
 
+      # pioarduino's fork of PlatformIO Core, which the pioarduino platform needs.
       - name: Install PlatformIO
-        run: pip install platformio
+        run: pip install pioarduino
 
       - name: Run native unit tests
         run: pio test -e native
@@ -40,7 +41,9 @@ jobs:
         # fwname: firmware-only binary for OTA updates
         # obname: onboard image (bootloader + partitions + firmware + littlefs merged)
         #         for flashing a new chip via USB/serial
-        # addr_bootloader: chip-dependent (ESP32: 0x1000, C3/S3/C6: 0x0)
+        # addr_bootloader: chip-dependent (ESP32: 0x1000, S3/C6: 0x0)
+        # No C3: on ESP-IDF 5.5 its image no longer fits the 1.5 MB OTA slot
+        # (docs/adr/0003-pioarduino-idf55.md).  C3 boards stay on 3.0.x.
         # addr_fs: must match spiffs/littlefs offset in esp32_3MB.csv
         include:
           - env: esp32dev
@@ -51,14 +54,6 @@ jobs:
             fwname: SomfyController.ino.esp32.bin
             fsname: SomfyController.littlefs.esp32.bin
             obname: SomfyController.onboard.esp32.bin
-          - env: esp32c3
-            name: ESP32-C3
-            chip: ESP32-C3
-            addr_bootloader: "0x0"
-            addr_fs: "0x310000"
-            fwname: SomfyController.ino.esp32c3.bin
-            fsname: SomfyController.littlefs.esp32c3.bin
-            obname: SomfyController.onboard.esp32c3.bin
           - env: esp32s3
             name: ESP32-S3
             chip: ESP32-S3
@@ -112,7 +107,7 @@ jobs:
           python-version: "3.12"
 
       - name: Install PlatformIO and esptool
-        run: pip install platformio esptool
+        run: pip install pioarduino esptool
 
       - name: Build LittleFS image
         run: pio run -e ${{ matrix.env }} -t buildfs
@@ -123,6 +118,9 @@ jobs:
       - name: Build firmware
         run: pio run -e ${{ matrix.env }}
 
+      - name: Check firmware size
+        run: python .github/size_check.py .pio/build/${{ matrix.env }} ${{ matrix.env }} 8
+
       - name: Restore LittleFS image
         run: cp littlefs.bin .pio/build/${{ matrix.env }}/littlefs.bin
 

AGENTS.md

@@ -2,3 +2,7 @@
 - Fix any test or type errors until the whole suite is green.
 - Add or update tests for the code you change, even if nobody asked.
 - to test run `pio test -e native`
+
+## Building
+- Firmware builds need pioarduino's Core (`pip install pioarduino`, or the pioarduino IDE extension). Stock PlatformIO Core refuses the pinned platform, see [ADR 0003](docs/adr/0003-pioarduino-idf55.md).
+- The platform installs its Python packages into `~/.platformio/penv` and fails on Windows while `pio.exe` from that env is running. Run builds as `python -m platformio` from that env's Python.

README.md

@@ -7,6 +7,9 @@
   * Backup does no longer use LITTLEFS. Only RAM is used. Storage should not wear out so quick.
   * Added uptime info in web UI
 
+## Building
+The firmware builds on [pioarduino](https://github.com/pioarduino/platform-espressif32) (Arduino 3.x on ESP-IDF 5.5), not the stock PlatformIO platform. In VS Code use the **pioarduino IDE** extension instead of PlatformIO IDE. On the command line, `pip install pioarduino` gives you `pio`. Run the unit tests with `pio test -e native`. [ADR 0003](docs/adr/0003-pioarduino-idf55.md) explains why.
+
 ## There is a partition layout modification it's just better to perform full ESP32 wipeout. Perform backup then wipeout the firmware and restore from the bkp.
   * fwname: firmware-only binary for OTA updates
   * obname: onboard image (bootloader + partitions + firmware + littlefs merged) for flashing a new chip via USB/serial

docs/SECURITY_PLAN.md

@@ -107,5 +107,6 @@ Everything else is checked with the `esp32devdbg` build, plus curl and websocket
 ## Not in this plan
 
 - TLS.
+- Checking GitHub's certificate in `GitOTA`. It calls `setInsecure()`, so anyone who can intercept the update download can serve their own firmware. ESP-IDF 5.5 ([ADR 0003](adr/0003-pioarduino-idf55.md)) ships a certificate bundle, so this is possible now. It needs its own change: the full bundle costs more flash than the ESP32 build has spare, and pinning GitHub's root certificates might fit.
 - Session tokens with expiry.
 - Contents of the binary backup file (it holds no passwords, but it does hold rolling codes; `/backup` becomes a config route).

docs/adr/0003-pioarduino-idf55.md

@@ -0,0 +1,18 @@
+# Firmware builds on pioarduino (Arduino 3.x, ESP-IDF 5.5), pinned to one release
+
+We moved every firmware env from `espressif32 @ 6.13.0` (Arduino 2.0.17 on ESP-IDF 4.4.7) to pioarduino's `55.03.312` (Arduino 3.3.12 on ESP-IDF 5.5). IDF 4.4 no longer gets security fixes for Wi-Fi, lwIP and mbedTLS, and the planned RMT radio needs IDF 5's RMT driver. The official PlatformIO platform never shipped Arduino 3.x, so pioarduino, a community fork, is the only way to get it in PlatformIO. We pin one release URL rather than `stable`, so a framework upgrade is always its own PR with its own size check and hardware test.
+
+## Considered Options
+
+- **Stay on IDF 4.4.** Rejected: no more security fixes, and no new RMT driver.
+- **Rebuild the IDF libraries with `custom_sdkconfig`** to leave out PPP, NAT and provisioning (about 20 KB the firmware never uses). Deferred: it makes builds much slower, and log level 2 already makes the ESP32 build fit.
+- **Grow the app slots in the partition table.** Rejected: an OTA update can't change the partition table, so every existing device would need a USB flash.
+
+## Consequences
+
+- **Firmware size.** The ESP32 image grows from 1,382,640 bytes (v3.0.14) to 1,555,472 at `CORE_DEBUG_LEVEL=2`, leaving 17 KB of the 1.5 MB OTA slot. At level 3 it doesn't fit, so release envs build at level 2 (warnings and errors) and debug envs stay at 3. PlatformIO's "Flash:" figure counts ELF sections, not the image, and reads about 45 KB low, so CI checks `firmware.bin` against the app partition and fails under 8 KB free (`.github/size_check.py`).
+- **No more C3 builds.** Its image grows by 214 KB to 1,651,216 bytes, 76.5 KB over the OTA slot even at level 2. C3 boards stay on the 3.0.x releases.
+- **Ethernet PHY ids are ours.** Arduino 3.x renumbered `eth_phy_type_t` (`ETH_PHY_GENERIC` is 0). NVS, backups and the web UI keep the Arduino 2.x numbers as `eth_phy_id_t` (`src/EthPhy.h`), translated by name when Ethernet starts. Never cast a stored id to the framework enum.
+- **Ethernet only on the original ESP32.** Arduino 3.x has the RMII MAC API only where `CONFIG_ETH_USE_ESP32_EMAC` is set. S3 and C6 build without Ethernet.
+- **Updates roll back until proven.** We override the core's `verifyRollbackLater()` hook to return true, and `loop()` marks the new firmware valid once it has run 60 s and the network is up. A reset before then boots the previous firmware. The LittleFS image an update wrote stays, so the old firmware runs with the new web UI.
+- **Tooling.** pioarduino needs its own Core fork (`pip install pioarduino`, the pioarduino IDE extension in VS Code), which it installs into `~/.platformio/penv` in place of PlatformIO Core.

platformio.ini

@@ -15,7 +15,9 @@ src_dir = src
 ; Shared by every firmware env (they extend it).  Kept out of [env] so the
 ; native test env doesn't inherit the Arduino framework and ESP32 libraries.
 [esp32]
-platform =  espressif32 @ 6.13.0
+; pioarduino: Arduino 3.3.12 on ESP-IDF 5.5.  Pinned to a release so an
+; upgrade is its own PR (docs/adr/0003-pioarduino-idf55.md).
+platform = https://github.com/pioarduino/platform-espressif32/releases/download/55.03.312/platform-espressif32.zip
 framework = arduino
 ; Unit tests only run on the PC: pio test -e native
 test_ignore = *
@@ -30,8 +32,9 @@ extra_scripts =
 	pre:minify.py
 	post:archive_elf.py
 board_build.filesystem = littlefs
+; Each env adds a CORE_DEBUG_LEVEL: 2 (warnings) for release builds, which
+; only fit the 1.5 MB OTA slot without the info logs, 3 for debug builds.
 build_flags =
-	-DCORE_DEBUG_LEVEL=3
 	-DCONFIG_ESP_COREDUMP_ENABLE_TO_FLASH=1
 	-DCONFIG_ESP_COREDUMP_DATA_FORMAT_ELF=1
 	-DCONFIG_ESP_COREDUMP_CHECKSUM_CRC32=1
@@ -57,7 +60,9 @@ build_flags =
 extends = esp32
 board = esp32dev
 board_build.partitions = esp32_3MB.csv
-build_flags = ${dualcore.build_flags}
+build_flags =
+	${dualcore.build_flags}
+	-DCORE_DEBUG_LEVEL=2
 
 [env:esp32devdbg]
 extends = esp32
@@ -73,39 +78,26 @@ board_build.partitions = esp32_huge_app.csv
 ; KBs of flash per file, so treat every file as large.
 build_flags =
 	${dualcore.build_flags}
+	-DCORE_DEBUG_LEVEL=3
 	--param=large-unit-insns=1000
 
-[env:esp32c3]
-extends = esp32
-board = esp32-c3-devkitm-1
-board_build.partitions = esp32_3MB.csv
-
 [env:esp32s3]
 extends = esp32
 board = esp32-s3-devkitc-1
-build_flags = ${dualcore.build_flags}
+build_flags =
+	${dualcore.build_flags}
+	-DCORE_DEBUG_LEVEL=2
 
+; Released as a debug build with one 3 MB app slot, so C6 boards update over
+; USB, not OTA.
 [env:esp32c6]
 extends = esp32
-platform = https://github.com/pioarduino/platform-espressif32/releases/download/stable/platform-espressif32.zip
 board = seeed_xiao_esp32c6
 build_type = debug
 board_build.partitions = esp32_huge_app.csv
 build_flags =
 	${esp32.build_flags}
-	-DCONFIG_FREERTOS_HZ=1000
+	-DCORE_DEBUG_LEVEL=3
-	-include "driver/gpio.h"
-lib_deps = 
-	; Low-Level libs, frameowrks and drivers
-	ESP32Async/AsyncTCP
-	https://github.com/Viproz/SmartRC-CC1101-Driver-Lib.git
-	; Commmunication Protocol
-	links2004/WebSockets
-	https://github.com/hmueller01/pubsubclient3.git
-	; Data-Parsing
-	bblanchon/ArduinoJson
-	; High-level wrappers/servers
-	ESP32Async/ESPAsyncWebServer
 
 ; Host-side unit tests: pio test -e native  (needs gcc/g++ on PATH).
 ; Only the sources the tests cover are built; test/stubs stands in for Arduino.h.

src/ESPNetwork.cpp

@@ -396,7 +396,7 @@ bool ESPNetwork::connectWired() {
     if(!ethPhyType(settings.Ethernet.phyType, phyType))
       ESP_LOGE(TAG, "Ethernet PHY type %u is not supported", (unsigned int)settings.Ethernet.phyType);
     else
-      begun = ETH.begin(settings.Ethernet.phyAddress, settings.Ethernet.PWRPin, settings.Ethernet.MDCPin, settings.Ethernet.MDIOPin, phyType, settings.Ethernet.CLKMode);
+      begun = ETH.begin(phyType, settings.Ethernet.phyAddress, settings.Ethernet.MDCPin, settings.Ethernet.MDIOPin, settings.Ethernet.PWRPin, settings.Ethernet.CLKMode);
     if(!begun) {
       ESP_LOGE(TAG, "Ethernet Begin failed");
       this->ethStarted = false;

src/LogLock.cpp

@@ -18,7 +18,6 @@
 // progress (see LogLockPolicy.h).
 #include <Arduino.h>
 #include <stdarg.h>
-#include "esp_idf_version.h"
 #include "freertos/FreeRTOS.h"
 #include "freertos/semphr.h"
 #include "freertos/task.h"
@@ -41,19 +40,12 @@ static volatile TaskHandle_t logOwner = nullptr;
 // line takes about 20ms at 115200 baud.
 #define LOG_LOCK_WAIT_MS 20
 
-#if ESP_IDF_VERSION_MAJOR >= 5
-#define taskOnCore(core) xTaskGetCurrentTaskHandleForCore(core)
-#define taskCore(task) xTaskGetCoreID(task)
-#else
-#define taskOnCore(core) xTaskGetCurrentTaskHandleForCPU(core)
-#define taskCore(task) xTaskGetAffinity(task)
-#endif
 static LogHolder logHolder(TaskHandle_t owner) {
   if(!owner) return LogHolder::unknown;
   for(BaseType_t core = 0; core < portNUM_PROCESSORS; core++)
-    if(taskOnCore(core) == owner) return LogHolder::running;
+    if(xTaskGetCurrentTaskHandleForCore(core) == owner) return LogHolder::running;
   // A task pinned elsewhere only runs once whatever preempted it there is done.
-  const BaseType_t core = taskCore(owner);
+  const BaseType_t core = xTaskGetCoreID(owner);
   if(core == tskNO_AFFINITY || core == xPortGetCoreID()) return LogHolder::preemptedHere;
   return LogHolder::preemptedElsewhere;
 }

src/SomfyController.ino

@@ -16,18 +16,20 @@
 #include "GitOTA.h"
 #include "esp_core_dump.h"
 #include "esp_system.h"
-#include "esp_idf_version.h"
-#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 0, 0)
 #include "esp_app_desc.h"
-#define getRunningElfSha esp_app_get_elf_sha256
-#else
 #include "esp_ota_ops.h"
-#define getRunningElfSha esp_ota_get_app_elf_sha256
-#endif
 #include "Timing.h"
 
 static const char *TAG = "Main";
 #define LOOP_SLOW_MS 100
+// How long an updated firmware has to run, and then have the network up, before
+// it is kept.  A reset before then rolls back to the previous firmware.
+#define OTA_PROVEN_MS 60000
+
+// Overrides the core's weak hook so it leaves an updated firmware unverified
+// at boot.  loop() confirms it once otaProven().
+extern "C" bool verifyRollbackLater() { return true; }
+static bool otaPending = false;
 
 ConfigSettings settings;
 Web webServer;
@@ -101,7 +103,7 @@ void setup() {
     static_assert(sizeof(crashedSha) <= sizeof(summary.app_elf_sha256), "copy would read past app_elf_sha256");
     SETCHARPROP(crashedSha, (const char *)summary.app_elf_sha256, sizeof(crashedSha));
     char runningSha[APP_ELF_SHA256_SZ] = {0};
-    getRunningElfSha(runningSha, sizeof(runningSha));
+    esp_app_get_elf_sha256(runningSha, sizeof(runningSha));
     if(strcmp(crashedSha, runningSha) == 0)
       CRASH_LOG("  Build: %s (running %s, same build)", crashedSha, runningSha);
     else
@@ -129,14 +131,15 @@ void setup() {
   net.setup();  
   somfy.begin();
   //git.checkForUpdate();
-#if ESP_ARDUINO_VERSION_MAJOR >= 3
+  // The core starts the watchdog with a 5s timeout before setup(), so
+  // esp_task_wdt_init() would fail and keep that.
   const esp_task_wdt_config_t wdt_config = { .timeout_ms = 15000, .idle_core_mask = 1, .trigger_panic = true };
-  esp_task_wdt_init(&wdt_config);
+  esp_task_wdt_reconfigure(&wdt_config);
-#else
-  esp_task_wdt_init(15, true); //enable panic so ESP32 restarts
-#endif
   esp_task_wdt_add(NULL); //add current thread to WDT watch
-
+  esp_ota_img_states_t otaState;
+  otaPending = esp_ota_get_state_partition(esp_ota_get_running_partition(), &otaState) == ESP_OK &&
+    otaState == ESP_OTA_IMG_PENDING_VERIFY;
+  if(otaPending) ESP_LOGW(TAG, "Updated firmware, keeping it once it has run %ds with the network up", OTA_PROVEN_MS / 1000);
 }
 
 void loop() {
@@ -183,6 +186,11 @@ void loop() {
     net.end();
     ESP.restart();
   }
+  if(otaPending && otaProven(net.connected(), millis(), OTA_PROVEN_MS)) {
+    otaPending = false;
+    if(esp_ota_mark_app_valid_cancel_rollback() == ESP_OK) ESP_LOGI(TAG, "Updated firmware confirmed, rollback cancelled");
+    else ESP_LOGE(TAG, "Unable to confirm the updated firmware");
+  }
   esp_task_wdt_reset();
   loopStats.record(pass.us());
 }

src/Utils.cpp

@@ -147,6 +147,9 @@ void RssiBroadcast::sent(bool broadcast, int rssi, int channel, uint32_t now) {
 bool worthRoaming(bool connected, int currentRssi, int apRssi, int minGain) {
   return !connected || apRssi >= currentRssi + minGain;
 }
+bool otaProven(bool networkUp, unsigned long uptimeMs, unsigned long minUptimeMs) {
+  return networkUp && uptimeMs >= minUptimeMs;
+}
 const char *wifiAuthModeName(uint8_t authMode) {
   switch(authMode) {
     case 0: return "OPEN";

src/Utils.h

@@ -82,6 +82,10 @@ enum class heap_emit_t : uint8_t { none, client, broadcast, room };
 // num is the socket client asked for, 255 for everyone.  A broadcast goes out
 // every intervalMs; in between only the clients in room 0 get one.
 heap_emit_t heapEmitTarget(uint8_t num, uint32_t sinceLastMs, uint32_t intervalMs, bool roomHasClients);
+// Whether a firmware the bootloader is holding for verification after an
+// update has proven itself: it has run for minUptimeMs and the network is
+// up.  Until then a reset rolls back to the previous firmware.
+bool otaProven(bool networkUp, unsigned long uptimeMs, unsigned long minUptimeMs);
 // How well loop() keeps up: the passes it made per second and its longest
 // pass, over the window since the last memStatus broadcast.  Only the loop
 // task calls record(), rate(), maxUs() and publish().  publish() closes the

test/test_utils/test_utils.cpp

@@ -257,6 +257,21 @@ static void test_worthRoaming_any_ap_while_disconnected() {
   TEST_ASSERT_TRUE(worthRoaming(false, 0, -90, 6));
 }
 
+/*********************************************************************
+ * otaProven
+ ********************************************************************/
+static void test_otaProven_after_the_minimum_uptime_with_network() {
+  TEST_ASSERT_TRUE(otaProven(true, 60000, 60000));
+  TEST_ASSERT_TRUE(otaProven(true, 3600000, 60000));
+}
+static void test_otaProven_not_before_the_minimum_uptime() {
+  TEST_ASSERT_FALSE(otaProven(true, 59999, 60000));
+}
+static void test_otaProven_not_without_network() {
+  // A firmware that can't connect would leave the device unreachable.
+  TEST_ASSERT_FALSE(otaProven(false, 3600000, 60000));
+}
+
 /*********************************************************************
  * wifiAuthModeName
  ********************************************************************/
@@ -384,6 +399,9 @@ int main(int argc, char **argv) {
   RUN_TEST(test_worthRoaming_needs_the_full_gain);
   RUN_TEST(test_worthRoaming_never_to_a_weaker_ap);
   RUN_TEST(test_worthRoaming_any_ap_while_disconnected);
+  RUN_TEST(test_otaProven_after_the_minimum_uptime_with_network);
+  RUN_TEST(test_otaProven_not_before_the_minimum_uptime);
+  RUN_TEST(test_otaProven_not_without_network);
   RUN_TEST(test_wifiAuthModeName_matches_the_idf_enum);
   RUN_TEST(test_wifiAuthModeName_unknown_past_the_shared_values);
   RUN_TEST(test_heapEmitTarget_one_client_always_gets_it);