From b525a800b03e8262aa5faa548e0e66901b5cd8a6 Mon Sep 17 00:00:00 2001 From: cjkas Date: Thu, 29 Oct 2026 16:59:53 +0100 Subject: [PATCH] test: e2e firmware update, and restarts told apart by boot time With --reboot, a restore of the radio alone brings it back as it was: on a board without Ethernet it read the radio from the wrong place. --firmware updates the device through /updateFirmware and checks it still runs a minute later, after it marks the update good. wait_for_restart compared uptimes, so a test run right after another restart could see more uptime than before and fail. It compares boot times, and waits while the device hasn't restarted yet. Co-Authored-By: Claude Opus 5.5 --- test/e2e/README.md | 3 ++- test/e2e/conftest.py | 1 + test/e2e/test_restore.py | 27 +++++++++++++++++++--- test/e2e/test_settings.py | 21 +++++++++++------ test/e2e/test_update.py | 48 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 89 insertions(+), 11 deletions(-) create mode 100644 test/e2e/test_update.py diff --git a/test/e2e/README.md b/test/e2e/README.md index f2086f1..4bdf4bd 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -12,7 +12,8 @@ Runs against a real device on your network, by hand, never in CI. It checks sock | `--password`, `--username` / `--pin` | the credentials of a Secured device | | `--radio --shade ` | also send real commands: that shade moves a little and back | | `--edit` | also add a room and delete it again | -| `--reboot` | also reboot the device and wait for it to come back, and restore the settings from a backup taken a moment before (it changes the NTP server, which the restore puts back) | +| `--reboot` | also reboot the device and wait for it to come back, and restore the settings from a backup taken a moment before (it changes the NTP server, which the restore puts back), and restore the radio settings alone | +| `--firmware ` | also update the device to that firmware, built for its chip, and check it is still running a minute later, after it marks the update good. Needs a build with an OTA slot | | `--lockout` | on a Secured device, also fail `/login` until this PC is refused, then wait out the 30 s lockout | | `--record` | save hello, the sync and the schema to `test/fixtures/proto2/`, for the web UI and Home Assistant tests | diff --git a/test/e2e/conftest.py b/test/e2e/conftest.py index 1b47f1e..b4ca78a 100644 --- a/test/e2e/conftest.py +++ b/test/e2e/conftest.py @@ -19,6 +19,7 @@ def pytest_addoption(parser): parser.addoption("--shade", type=int, default=None, help="with --radio: the id of the shade the commands may move") parser.addoption("--edit", action="store_true", help="also run the edits that change the shade model, then undo them") parser.addoption("--reboot", action="store_true", help="also run the tests that reboot the device") + parser.addoption("--firmware", default=None, help="also update the device to this firmware.bin, built for its chip") parser.addoption("--lockout", action="store_true", help="also run the test that locks this PC out of /login for 30 s") diff --git a/test/e2e/test_restore.py b/test/e2e/test_restore.py index 96593b7..45e9ff9 100644 --- a/test/e2e/test_restore.py +++ b/test/e2e/test_restore.py @@ -9,7 +9,7 @@ import pytest from proto_client import run_sync from test_commands import is_ok, op, synced_model from test_routes import auth, http, restore_without_a_file, upload -from test_settings import desc, settings, uptime, wait_for_restart +from test_settings import boot_time, desc, settings, wait_for_restart # Saved for the round trip and put back by the restore. .invalid never resolves. CHANGED_NTP_SERVER = "restore.e2e.invalid" @@ -50,7 +50,7 @@ def test_a_restore_brings_back_the_parts_that_were_ticked(request, host, token, pytest.skip("restores a backup and reboots the device: pass --reboot") c = connect(schema=schema.hash) _, frames, _ = run_sync(c) - uptime_before = uptime(schema, frames) + booted = boot_time(schema, frames) general = settings(c, schema, "general") ntp = {name: general.get(name, schema.default(name)) for name in ("ntpServer", "posixZone")} assert ntp["ntpServer"] != CHANGED_NTP_SERVER @@ -63,10 +63,31 @@ def test_a_restore_brings_back_the_parts_that_were_ticked(request, host, token, options = json.dumps({"shades": False, "settings": True}) status, _, body = upload(host, "/restore", auth(token), file=backup, fields={"data": options}) assert status == 200, body - wait_for_restart(host, token, schema, uptime_before) + wait_for_restart(host, token, schema, booted) finally: c, _, _ = synced_model(connect, schema) restored = settings(c, schema, "general").get("ntpServer", schema.default("ntpServer")) if restored != ntp["ntpServer"]: op(c, schema, "saveSettings", dict(ntp, section="general")) assert restored == ntp["ntpServer"] + + +def test_a_restore_of_the_radio_alone_brings_it_back_as_it_was(request, host, token, connect, schema): + """4.0 boards without Ethernet declared their network record 28 bytes + longer than they wrote, so a restore that skipped it read the radio from + the middle of its record.""" + if not request.config.getoption("--reboot"): + pytest.skip("restores a backup and reboots the device: pass --reboot") + c = connect(schema=schema.hash) + _, frames, _ = run_sync(c) + booted = boot_time(schema, frames) + radio = settings(c, schema, "radio") + status, _, backup = http(host, "/backup", headers=auth(token)) + assert status == 200 + + options = json.dumps({"shades": False, "transceiver": True}) + status, _, body = upload(host, "/restore", auth(token), file=backup, fields={"data": options}) + assert status == 200, body + wait_for_restart(host, token, schema, booted) + c, _, _ = synced_model(connect, schema) + assert settings(c, schema, "radio") == radio diff --git a/test/e2e/test_settings.py b/test/e2e/test_settings.py index 45d23f9..e04d99e 100644 --- a/test/e2e/test_settings.py +++ b/test/e2e/test_settings.py @@ -120,20 +120,20 @@ def test_reboot(request, host, token, connect, schema): pytest.skip("reboots the device: pass --reboot") c = connect(schema=schema.hash) _, frames, _ = run_sync(c) - uptime_before = uptime(schema, frames) + booted = boot_time(schema, frames) assert is_ok(schema, op(c, schema, "reboot", {})) assert c.closed.wait(15), "the socket stayed open" - wait_for_restart(host, token, schema, uptime_before) + wait_for_restart(host, token, schema, booted) -def wait_for_restart(host, token, schema, uptime_before): - """Waits until the device is back, and checks it restarted.""" +def wait_for_restart(host, token, schema, booted): + """Waits until the device has restarted since it booted at booted (boot_time).""" # The device closes its sockets just before it restarts, and may still # take a connection in between: that one never gets its hello. time.sleep(3) end = time.time() + 90 while True: - assert time.time() < end, "the device didn't come back" + assert time.time() < end, "the device didn't restart" try: back = ProtoClient(host, params={"apikey": token} if token else None, timeout=3) except OSError: @@ -143,8 +143,10 @@ def wait_for_restart(host, token, schema, uptime_before): if back.status == 101: hello, frames, _ = run_sync(back, timeout=10) assert hello["d"]["sh"] == schema.hash - assert uptime(schema, frames) < uptime_before, "the device didn't restart" - return + # Compared as boot times: a device that restarted a moment + # ago may have had less uptime than it has now. + if boot_time(schema, frames) > booted + 2: + return except TimeoutError: pass # not back yet finally: @@ -152,6 +154,11 @@ def wait_for_restart(host, token, schema, uptime_before): time.sleep(2) +def boot_time(schema, frames): + """When the device booted, as a time.time(), from the status a sync ends with.""" + return time.time() - uptime(schema, frames) / 1000 + + def uptime(schema, frames): """The uptime, in ms, from the status a sync ends with.""" key = schema.keys["status"] diff --git a/test/e2e/test_update.py b/test/e2e/test_update.py new file mode 100644 index 0000000..d3c1a2c --- /dev/null +++ b/test/e2e/test_update.py @@ -0,0 +1,48 @@ +"""A firmware update through /updateFirmware, as the web UI sends it. + +Pass --firmware with a firmware.bin built for the device's chip, e.g. +.pio/build/esp32s3/firmware.bin. The device must have an OTA slot: the +release builds do, the C5, C6 and debug builds don't. +""" +import time + +import pytest + +from proto_client import run_sync +from test_routes import auth, upload +from test_settings import boot_time, wait_for_restart + +# The new firmware is marked good once it has run this long with the network +# up; a reset before then boots the old one (docs/adr/0003-pioarduino-idf55.md). +PROVEN_AFTER_S = 60 + + +@pytest.fixture +def firmware(request): + path = request.config.getoption("--firmware") + if not path: + pytest.skip("updates the device's firmware: pass --firmware ") + with open(path, "rb") as f: + return f.read() + + +def test_an_update_restarts_on_the_new_firmware_and_stays(host, token, connect, schema, firmware): + c = connect(schema=schema.hash) + _, frames, _ = run_sync(c) + booted = boot_time(schema, frames) + c.close() + + status, _, body = upload(host, "/updateFirmware", auth(token), file=firmware) + assert status == 200, body + wait_for_restart(host, token, schema, booted) + + # Past the point where it is marked good it is still running, and hasn't + # restarted on the old firmware. + c = connect(schema=schema.hash) + _, frames, _ = run_sync(c) + restarted = boot_time(schema, frames) + c.close() + time.sleep(PROVEN_AFTER_S + 10) + c = connect(schema=schema.hash) + _, frames, _ = run_sync(c) + assert abs(boot_time(schema, frames) - restarted) < 5, "the device restarted again"