test: e2e firmware update, and restarts told apart by boot time

With --reboot, a restore of the radio alone brings it back as it was:
on a board without Ethernet it read the radio from the wrong place.

--firmware <firmware.bin> updates the device through /updateFirmware and
checks it still runs a minute later, after it marks the update good.

wait_for_restart compared uptimes, so a test run right after another
restart could see more uptime than before and fail.  It compares boot
times, and waits while the device hasn't restarted yet.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

test/e2e/README.md

@@ -12,7 +12,8 @@ Runs against a real device on your network, by hand, never in CI. It checks sock
 | `--password`, `--username` / `--pin` | the credentials of a Secured device |
 | `--radio --shade <id>` | also send real commands: that shade moves a little and back |
 | `--edit` | also add a room and delete it again |
-| `--reboot` | also reboot the device and wait for it to come back, and restore the settings from a backup taken a moment before (it changes the NTP server, which the restore puts back) |
+| `--reboot` | also reboot the device and wait for it to come back, and restore the settings from a backup taken a moment before (it changes the NTP server, which the restore puts back), and restore the radio settings alone |
+| `--firmware <firmware.bin>` | also update the device to that firmware, built for its chip, and check it is still running a minute later, after it marks the update good. Needs a build with an OTA slot |
 | `--lockout` | on a Secured device, also fail `/login` until this PC is refused, then wait out the 30 s lockout |
 | `--record` | save hello, the sync and the schema to `test/fixtures/proto2/`, for the web UI and Home Assistant tests |
 

test/e2e/conftest.py

@@ -19,6 +19,7 @@ def pytest_addoption(parser):
     parser.addoption("--shade", type=int, default=None, help="with --radio: the id of the shade the commands may move")
     parser.addoption("--edit", action="store_true", help="also run the edits that change the shade model, then undo them")
     parser.addoption("--reboot", action="store_true", help="also run the tests that reboot the device")
+    parser.addoption("--firmware", default=None, help="also update the device to this firmware.bin, built for its chip")
     parser.addoption("--lockout", action="store_true", help="also run the test that locks this PC out of /login for 30 s")
 
 

test/e2e/test_restore.py

@@ -9,7 +9,7 @@ import pytest
 from proto_client import run_sync
 from test_commands import is_ok, op, synced_model
 from test_routes import auth, http, restore_without_a_file, upload
-from test_settings import desc, settings, uptime, wait_for_restart
+from test_settings import boot_time, desc, settings, wait_for_restart
 
 # Saved for the round trip and put back by the restore.  .invalid never resolves.
 CHANGED_NTP_SERVER = "restore.e2e.invalid"
@@ -50,7 +50,7 @@ def test_a_restore_brings_back_the_parts_that_were_ticked(request, host, token,
         pytest.skip("restores a backup and reboots the device: pass --reboot")
     c = connect(schema=schema.hash)
     _, frames, _ = run_sync(c)
-    uptime_before = uptime(schema, frames)
+    booted = boot_time(schema, frames)
     general = settings(c, schema, "general")
     ntp = {name: general.get(name, schema.default(name)) for name in ("ntpServer", "posixZone")}
     assert ntp["ntpServer"] != CHANGED_NTP_SERVER
@@ -63,10 +63,31 @@ def test_a_restore_brings_back_the_parts_that_were_ticked(request, host, token,
         options = json.dumps({"shades": False, "settings": True})
         status, _, body = upload(host, "/restore", auth(token), file=backup, fields={"data": options})
         assert status == 200, body
-        wait_for_restart(host, token, schema, uptime_before)
+        wait_for_restart(host, token, schema, booted)
     finally:
         c, _, _ = synced_model(connect, schema)
         restored = settings(c, schema, "general").get("ntpServer", schema.default("ntpServer"))
         if restored != ntp["ntpServer"]:
             op(c, schema, "saveSettings", dict(ntp, section="general"))
     assert restored == ntp["ntpServer"]
+
+
+def test_a_restore_of_the_radio_alone_brings_it_back_as_it_was(request, host, token, connect, schema):
+    """4.0 boards without Ethernet declared their network record 28 bytes
+    longer than they wrote, so a restore that skipped it read the radio from
+    the middle of its record."""
+    if not request.config.getoption("--reboot"):
+        pytest.skip("restores a backup and reboots the device: pass --reboot")
+    c = connect(schema=schema.hash)
+    _, frames, _ = run_sync(c)
+    booted = boot_time(schema, frames)
+    radio = settings(c, schema, "radio")
+    status, _, backup = http(host, "/backup", headers=auth(token))
+    assert status == 200
+
+    options = json.dumps({"shades": False, "transceiver": True})
+    status, _, body = upload(host, "/restore", auth(token), file=backup, fields={"data": options})
+    assert status == 200, body
+    wait_for_restart(host, token, schema, booted)
+    c, _, _ = synced_model(connect, schema)
+    assert settings(c, schema, "radio") == radio

test/e2e/test_settings.py

@@ -120,20 +120,20 @@ def test_reboot(request, host, token, connect, schema):
         pytest.skip("reboots the device: pass --reboot")
     c = connect(schema=schema.hash)
     _, frames, _ = run_sync(c)
-    uptime_before = uptime(schema, frames)
+    booted = boot_time(schema, frames)
     assert is_ok(schema, op(c, schema, "reboot", {}))
     assert c.closed.wait(15), "the socket stayed open"
-    wait_for_restart(host, token, schema, uptime_before)
+    wait_for_restart(host, token, schema, booted)
 
 
-def wait_for_restart(host, token, schema, uptime_before):
+def wait_for_restart(host, token, schema, booted):
-    """Waits until the device is back, and checks it restarted."""
+    """Waits until the device has restarted since it booted at booted (boot_time)."""
     # The device closes its sockets just before it restarts, and may still
     # take a connection in between: that one never gets its hello.
     time.sleep(3)
     end = time.time() + 90
     while True:
-        assert time.time() < end, "the device didn't come back"
+        assert time.time() < end, "the device didn't restart"
         try:
             back = ProtoClient(host, params={"apikey": token} if token else None, timeout=3)
         except OSError:
@@ -143,8 +143,10 @@ def wait_for_restart(host, token, schema, uptime_before):
             if back.status == 101:
                 hello, frames, _ = run_sync(back, timeout=10)
                 assert hello["d"]["sh"] == schema.hash
-                assert uptime(schema, frames) < uptime_before, "the device didn't restart"
+                # Compared as boot times: a device that restarted a moment
-                return
+                # ago may have had less uptime than it has now.
+                if boot_time(schema, frames) > booted + 2:
+                    return
         except TimeoutError:
             pass  # not back yet
         finally:
@@ -152,6 +154,11 @@ def wait_for_restart(host, token, schema, uptime_before):
         time.sleep(2)
 
 
+def boot_time(schema, frames):
+    """When the device booted, as a time.time(), from the status a sync ends with."""
+    return time.time() - uptime(schema, frames) / 1000
+
+
 def uptime(schema, frames):
     """The uptime, in ms, from the status a sync ends with."""
     key = schema.keys["status"]

test/e2e/test_update.py

@@ -0,0 +1,48 @@
+"""A firmware update through /updateFirmware, as the web UI sends it.
+
+Pass --firmware with a firmware.bin built for the device's chip, e.g.
+.pio/build/esp32s3/firmware.bin.  The device must have an OTA slot: the
+release builds do, the C5, C6 and debug builds don't.
+"""
+import time
+
+import pytest
+
+from proto_client import run_sync
+from test_routes import auth, upload
+from test_settings import boot_time, wait_for_restart
+
+# The new firmware is marked good once it has run this long with the network
+# up; a reset before then boots the old one (docs/adr/0003-pioarduino-idf55.md).
+PROVEN_AFTER_S = 60
+
+
+@pytest.fixture
+def firmware(request):
+    path = request.config.getoption("--firmware")
+    if not path:
+        pytest.skip("updates the device's firmware: pass --firmware <firmware.bin>")
+    with open(path, "rb") as f:
+        return f.read()
+
+
+def test_an_update_restarts_on_the_new_firmware_and_stays(host, token, connect, schema, firmware):
+    c = connect(schema=schema.hash)
+    _, frames, _ = run_sync(c)
+    booted = boot_time(schema, frames)
+    c.close()
+
+    status, _, body = upload(host, "/updateFirmware", auth(token), file=firmware)
+    assert status == 200, body
+    wait_for_restart(host, token, schema, booted)
+
+    # Past the point where it is marked good it is still running, and hasn't
+    # restarted on the old firmware.
+    c = connect(schema=schema.hash)
+    _, frames, _ = run_sync(c)
+    restarted = boot_time(schema, frames)
+    c.close()
+    time.sleep(PROVEN_AFTER_S + 10)
+    c = connect(schema=schema.hash)
+    _, frames, _ = run_sync(c)
+    assert abs(boot_time(schema, frames) - restarted) < 5, "the device restarted again"