fix: review fixes for the shades.cfg records

- A group in a backup from before version 23 has no rolling code, and a
  restore wrote 0 to NVS for it.  It keeps the code NVS has.
- A restore reads each record into one that starts as the shade, group
  or settings it replaces, so what an older version lacks stays as it
  was (the update check below version 20 among them), as before.
- restoreFile and loadFile read the rooms, shades and groups through one
  readModel(); the settings, network and radio are applied by helpers.
- The shade's tilt is forgotten where the shades are written, not in the
  conversion to a record.
- rollingCodeAfterLoad moves to RollingCode.h and its tests to
  test_rollingcode.
- The topics a deleted shade or group clears are one list each
  (MqttTopics.h), tested against what is published.
- e2e: a restore of the shades brings back every shade and group, and
  the README has the rollback check by hand.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

test/e2e/README.md

@@ -21,4 +21,11 @@ A plain run sends nothing on air, changes no setting (it saves only values the d
 
 The web UI holds one of the device's 5 socket slots for each open tab, and Home Assistant holds one. The tests that fill every slot fill the ones left, so close the UI's tabs for the full load.
 
+A rollback is checked by hand, on a release build with an OTA slot:
+
+1. Note the version the web UI shows.
+2. Build firmware that restarts before it is marked good: add `abort();` at the end of `setup()` in `src/SomfyController.cpp`, and change `data-src/appversion` so the two versions differ.
+3. Update to it through the web UI, with the serial monitor open.
+4. The crash restarts the device, and the bootloader boots the old firmware ([ADR 0003](../../docs/adr/0003-pioarduino-idf55.md)). The web UI shows the version from step 1 again.
+
 The tests run in whatever security mode the device is in; the suite never changes it. To test both modes, run the suite once in each.

test/e2e/test_restore.py

@@ -91,3 +91,22 @@ def test_a_restore_of_the_radio_alone_brings_it_back_as_it_was(request, host, to
     wait_for_restart(host, token, schema, booted)
     c, _, _ = synced_model(connect, schema)
     assert settings(c, schema, "radio") == radio
+
+
+def test_a_restore_of_the_shades_brings_them_back_as_they_were(request, host, token, connect, schema):
+    """Every shade and group, their rolling codes included, from a backup taken a moment before."""
+    if not request.config.getoption("--reboot"):
+        pytest.skip("restores a backup and reboots the device: pass --reboot")
+    _, shades, groups = synced_model(connect, schema)
+    _, frames, _ = run_sync(connect(schema=schema.hash))
+    booted = boot_time(schema, frames)
+    status, _, backup = http(host, "/backup", headers=auth(token))
+    assert status == 200
+
+    options = json.dumps({"shades": True})
+    status, _, body = upload(host, "/restore", auth(token), file=backup, fields={"data": options})
+    assert status == 200, body
+    wait_for_restart(host, token, schema, booted)
+    _, shades_after, groups_after = synced_model(connect, schema)
+    assert shades_after == shades
+    assert groups_after == groups

test/e2e/test_update.py

@@ -3,6 +3,10 @@
 Pass --firmware with a firmware.bin built for the device's chip, e.g.
 .pio/build/esp32s3/firmware.bin.  The device must have an OTA slot: the
 release builds do, the C5, C6 and debug builds don't.
+
+It can't tell the new firmware from the old when both have the same
+version, and it doesn't make the update fail.  test/e2e/README.md has the
+rollback check by hand.
 """
 import time
 

test/test_configrecords/test_configrecords.cpp

@@ -272,7 +272,7 @@ static void test_a_room_reads_back() {
   writeRoomRecord(w, room);
   TextReader r(w.text);
   room_record_t got;
-  readRoomRecord(r, header(), got);
+  readRoomRecord(r, got);
   TEST_ASSERT_EQUAL_UINT8(2, got.roomId);
   TEST_ASSERT_EQUAL_STRING("Living room", got.name);
   TEST_ASSERT_EQUAL_INT8(5, got.sortOrder);
@@ -401,7 +401,7 @@ static void test_the_longest_name_reads_back() {
   writeRoomRecord(w, room);
   TextReader r(w.text);
   room_record_t got;
-  readRoomRecord(r, header(), got);
+  readRoomRecord(r, got);
   TEST_ASSERT_EQUAL_STRING("12345678901234567890", got.name);
   TEST_ASSERT_EQUAL_INT8(7, got.sortOrder);
 }
@@ -506,8 +506,8 @@ static void test_values_a_newer_version_added_are_skipped() {
   writeRoomRecord(w, room_record_t{2, "Attic", 1});
   TextReader r(w.text);
   room_record_t got;
-  readRoomRecord(r, header(), got);
-  readRoomRecord(r, header(), got);
+  readRoomRecord(r, got);
+  readRoomRecord(r, got);
   TEST_ASSERT_EQUAL_UINT8(2, got.roomId);
   TEST_ASSERT_EQUAL_STRING("Attic", got.name);
 }
@@ -522,11 +522,11 @@ static void test_a_short_record_never_reads_the_next() {
   writeRoomRecord(w, room_record_t{2, "Attic", 1});
   TextReader r(text + w.text);
   room_record_t got;
-  readRoomRecord(r, header(), got);
+  readRoomRecord(r, got);
   TEST_ASSERT_EQUAL_UINT8(1, got.roomId);
   TEST_ASSERT_EQUAL_STRING("Hall", got.name);
   TEST_ASSERT_EQUAL_INT8(0, got.sortOrder);
-  readRoomRecord(r, header(), got);
+  readRoomRecord(r, got);
   TEST_ASSERT_EQUAL_UINT8(2, got.roomId);
   TEST_ASSERT_EQUAL_INT8(1, got.sortOrder);
 }
@@ -538,7 +538,7 @@ static void test_skipping_records_passes_their_newlines() {
   TextReader r(w.text);
   TEST_ASSERT_TRUE(skipRecords(r, 2));
   room_record_t got;
-  readRoomRecord(r, header(), got);
+  readRoomRecord(r, got);
   TEST_ASSERT_EQUAL_STRING("Last", got.name);
   TEST_ASSERT_FALSE(skipRecords(r, 1));
 }
@@ -551,7 +551,7 @@ static void writeShadeStart(TextWriter &w, uint8_t id) {
   w.writeUInt32(0x111111);
   w.writeString("Old", CFG_NAME_LEN);
 }
-static void test_a_version_4_shade() {
+static std::string version4Shade() {
   TextWriter w;
   writeShadeStart(w, 2);
   w.writeUInt8(0);        // tiltType
@@ -565,23 +565,62 @@ static void test_a_version_4_shade() {
   w.writeFloat(-1, 5);    // myTiltPos
   w.writeFloat(25, 5);    // currentPos
   w.writeFloat(0, 5, CFG_REC_END);
-  TextReader r(w.text);
+  return w.text;
+}
+static void test_a_version_4_shade() {
+  const std::string text = version4Shade();
+  TextReader r(text);
   shade_record_t got;
   readShadeRecord(r, header(4), got);
   TEST_ASSERT_EQUAL_UINT8(2, got.shadeId);
+  TEST_ASSERT_TRUE(got.paired);
+  TEST_ASSERT_EQUAL_UINT8(static_cast<uint8_t>(shade_types::roller), static_cast<uint8_t>(got.shadeType));
+  TEST_ASSERT_EQUAL_HEX32(0x111111, got.remoteAddress);
   TEST_ASSERT_EQUAL_STRING("Old", got.name);
+  TEST_ASSERT_EQUAL_UINT8(static_cast<uint8_t>(tilt_types::none), static_cast<uint8_t>(got.tiltType));
   TEST_ASSERT_EQUAL_UINT8(static_cast<uint8_t>(radio_proto::RTS), static_cast<uint8_t>(got.proto));
+  TEST_ASSERT_EQUAL_UINT8(56, got.bitLength);
   TEST_ASSERT_EQUAL_UINT32(15000, got.upTime);
+  TEST_ASSERT_EQUAL_UINT32(14000, got.downTime);
+  TEST_ASSERT_EQUAL_UINT32(0, got.tiltTime);
   TEST_ASSERT_EQUAL_UINT16(100, got.stepSize);
   TEST_ASSERT_EQUAL_HEX32(0x222222, got.linkedRemotes[0]);
+  TEST_ASSERT_EQUAL_HEX32(0, got.linkedRemotes[4]);
   TEST_ASSERT_EQUAL_HEX32(0, got.linkedRemotes[5]);
   TEST_ASSERT_EQUAL_UINT16(300, got.lastRollingCode);
+  TEST_ASSERT_EQUAL_HEX8(0, got.flags);
   TEST_ASSERT_EQUAL_FLOAT(50, got.myPos);
+  TEST_ASSERT_EQUAL_FLOAT(-1, got.myTiltPos);
   TEST_ASSERT_EQUAL_FLOAT(25, got.currentPos);
+  TEST_ASSERT_EQUAL_FLOAT(0, got.currentTiltPos);
+  TEST_ASSERT_FALSE(got.flipCommands);
+  TEST_ASSERT_FALSE(got.flipPosition);
   TEST_ASSERT_EQUAL_UINT8(1, got.repeats);
   TEST_ASSERT_EQUAL_INT8(1, got.sortOrder);
+  TEST_ASSERT_EQUAL_UINT8(0, got.gpioUp);
   TEST_ASSERT_EQUAL_UINT8(0, got.roomId);
-  TEST_ASSERT_EQUAL(w.text.size(), r.position());
+  TEST_ASSERT_EQUAL(text.size(), r.position());
+}
+// A restore reads into a record that starts as the shade it replaces, and
+// what an older version lacks stays as it was.
+static void test_what_an_older_version_lacks_stays_as_it_was() {
+  const std::string text = version4Shade();
+  TextReader r(text);
+  shade_record_t got = aShade();
+  readShadeRecord(r, header(4), got);
+  TEST_ASSERT_EQUAL_UINT32(15000, got.upTime);
+  TEST_ASSERT_EQUAL_HEX32(0x222222, got.linkedRemotes[0]);
+  TEST_ASSERT_EQUAL_UINT8(static_cast<uint8_t>(radio_proto::GP_Remote), static_cast<uint8_t>(got.proto));
+  TEST_ASSERT_EQUAL_UINT16(250, got.stepSize);
+  TEST_ASSERT_EQUAL_HEX32(0x200005, got.linkedRemotes[5]);
+  TEST_ASSERT_EQUAL_HEX8(0x21, got.flags);
+  TEST_ASSERT_TRUE(got.flipCommands);
+  TEST_ASSERT_TRUE(got.flipPosition);
+  TEST_ASSERT_EQUAL_UINT8(4, got.repeats);
+  TEST_ASSERT_EQUAL_UINT8(25, got.gpioUp);
+  TEST_ASSERT_EQUAL_UINT8(27, got.gpioMy);
+  TEST_ASSERT_EQUAL_UINT8(2, got.gpioFlags);
+  TEST_ASSERT_EQUAL_UINT8(5, got.roomId);
 }
 // Version 2 saved positions as fractions and the favorite as a byte.
 static void test_a_version_2_shade_has_fractions() {
@@ -664,6 +703,7 @@ static void test_a_version_23_group() {
   group_record_t got;
   readGroupRecord(r, header(23), got);
   TEST_ASSERT_EQUAL_UINT16(4321, got.lastRollingCode);
+  TEST_ASSERT_TRUE(got.hasRollingCode);
   TEST_ASSERT_EQUAL_UINT8(2, got.linkedShades[1]);
   TEST_ASSERT_EQUAL_UINT8(6, got.roomId);
   TEST_ASSERT_EQUAL(w.text.size(), r.position());
@@ -681,14 +721,34 @@ static void test_a_version_11_group() {
   TEST_ASSERT_FALSE(got.flipCommands);
   TEST_ASSERT_EQUAL_UINT16(0, got.lastRollingCode);
 }
-// Before version 20 the settings had no update check.
-static void test_version_19_settings_check_for_updates() {
+// Before version 23 a group had no rolling code, so a restore leaves the one
+// NVS holds.
+static void test_a_group_before_version_23_has_no_rolling_code() {
+  TextWriter w;
+  writeGroupStart(w);
+  for(uint8_t j = 0; j < SOMFY_MAX_GROUPED_SHADES; j++) w.writeUInt8(0, j == SOMFY_MAX_GROUPED_SHADES - 1 ? CFG_REC_END : CFG_VALUE_SEP);
+  TextReader r(w.text);
+  group_record_t got = aGroup();
+  readGroupRecord(r, header(11), got);
+  TEST_ASSERT_FALSE(got.hasRollingCode);
+  TEST_ASSERT_EQUAL_UINT16(1234, got.lastRollingCode);
+  TEST_ASSERT_TRUE(got.flipCommands);
+  TEST_ASSERT_EQUAL_UINT8(4, got.roomId);
+  TextWriter v24;
+  writeGroupRecord(v24, aGroup());
+  TextReader r24(v24.text);
+  readGroupRecord(r24, header(), got);
+  TEST_ASSERT_TRUE(got.hasRollingCode);
+}
+// Before version 20 the settings had no update check, which stays as it was.
+static void test_version_19_settings_leave_the_update_check() {
   TextReader r("\"v2.0.0\",\"host\",\"pool.ntp.org\",\"\",true \n");
   settings_record_t got;
+  got.checkForUpdate = false;
   readSettingsRecord(r, header(19), got);
   TEST_ASSERT_EQUAL_STRING("host", got.hostname);
   TEST_ASSERT_TRUE(got.ssdpBroadcast);
-  TEST_ASSERT_TRUE(got.checkForUpdate);
+  TEST_ASSERT_FALSE(got.checkForUpdate);
 }
 // Before version 22 the network record had no MQTT settings.
 static void test_a_version_21_network_has_no_mqtt() {
@@ -703,15 +763,6 @@ static void test_a_version_21_network_has_no_mqtt() {
   TEST_ASSERT_EQUAL_UINT8(9, r.readUInt8(0));
 }
 
-static void test_a_load_keeps_the_higher_rolling_code() {
-  TEST_ASSERT_EQUAL_UINT16(500, rollingCodeAfterLoad(500, 300, false));
-  TEST_ASSERT_EQUAL_UINT16(700, rollingCodeAfterLoad(500, 700, false));
-}
-static void test_a_restore_takes_the_backups_rolling_code() {
-  TEST_ASSERT_EQUAL_UINT16(300, rollingCodeAfterLoad(500, 300, true));
-  TEST_ASSERT_EQUAL_UINT16(700, rollingCodeAfterLoad(500, 700, true));
-}
-
 // A real backup, which isn't in the repo: it holds the remotes' addresses.
 // CONFIG_BACKUP=<path> pio test -e native -f test_configrecords
 static void test_a_real_backup_reads_to_its_end() {
@@ -727,7 +778,7 @@ static void test_a_real_backup_reads_to_its_end() {
   TEST_ASSERT_EQUAL(static_cast<uint32_t>(h.length), r.position());
   for(uint8_t i = 0; i < h.roomRecords; i++) {
     room_record_t rec;
-    readRoomRecord(r, h, rec);
+    readRoomRecord(r, rec);
     TEST_ASSERT_NOT_EQUAL(0, rec.roomId);
   }
   for(uint8_t i = 0; i < h.shadeRecords; i++) {
@@ -786,14 +837,14 @@ int main(int argc, char **argv) {
   RUN_TEST(test_a_short_record_never_reads_the_next);
   RUN_TEST(test_skipping_records_passes_their_newlines);
   RUN_TEST(test_a_version_4_shade);
+  RUN_TEST(test_what_an_older_version_lacks_stays_as_it_was);
   RUN_TEST(test_a_version_2_shade_has_fractions);
   RUN_TEST(test_a_version_13_shade);
   RUN_TEST(test_a_version_23_group);
   RUN_TEST(test_a_version_11_group);
-  RUN_TEST(test_version_19_settings_check_for_updates);
+  RUN_TEST(test_a_group_before_version_23_has_no_rolling_code);
+  RUN_TEST(test_version_19_settings_leave_the_update_check);
   RUN_TEST(test_a_version_21_network_has_no_mqtt);
-  RUN_TEST(test_a_load_keeps_the_higher_rolling_code);
-  RUN_TEST(test_a_restore_takes_the_backups_rolling_code);
   RUN_TEST(test_a_real_backup_reads_to_its_end);
   return UNITY_END();
 }

test/test_mqttdisco/test_mqttdisco.cpp

@@ -1,5 +1,9 @@
 #include <unity.h>
+#include <string>
+#include <vector>
 #include "MqttDisco.h"
+#include "MqttTopics.h"
+#include "ShadeChanges.h"
 
 void setUp() {}
 void tearDown() {}
@@ -170,6 +174,29 @@ static void test_the_config_topic_of_each_kind() {
   TEST_ASSERT_FALSE(isDiscoSwitch(shade_types::garage1));
 }
 
+static bool hasTopic(const char *const *topics, size_t count, const std::string &topic) {
+  for(size_t i = 0; i < count; i++) if(topic == topics[i]) return true;
+  return false;
+}
+static void keepRetained(const char *topic, int32_t value, bool retain, void *ctx) {
+  (void)value;
+  if(retain) static_cast<std::vector<std::string> *>(ctx)->push_back(topic);
+}
+// A deleted shade leaves none of its topics at the broker.
+static void test_deleting_a_shade_clears_every_state_topic_the_broker_keeps() {
+  shade_snapshot_t shade{};
+  std::vector<std::string> retained;
+  publishShadeTopics(SHADE_CHANGED_TOPICS, shade, keepRetained, &retained);
+  TEST_ASSERT_FALSE(retained.empty());
+  for(const std::string &topic : retained)
+    TEST_ASSERT_TRUE_MESSAGE(hasTopic(SHADE_TOPICS, SHADE_TOPIC_COUNT, topic), topic.c_str());
+}
+// Group unpublish named it SunSensor, so a deleted group's stayed.
+static void test_deleting_a_group_clears_its_sun_sensor_topic() {
+  TEST_ASSERT_TRUE(hasTopic(GROUP_TOPICS, GROUP_TOPIC_COUNT, "sunSensor"));
+  TEST_ASSERT_FALSE(hasTopic(GROUP_TOPICS, GROUP_TOPIC_COUNT, "SunSensor"));
+}
+
 int main(int argc, char **argv) {
   UNITY_BEGIN();
   RUN_TEST(test_a_roller_is_a_cover_under_the_discovery_topic);
@@ -183,5 +210,7 @@ int main(int argc, char **argv) {
   RUN_TEST(test_a_tilt_only_blind_has_only_the_tilt);
   RUN_TEST(test_a_dry_contact_is_a_switch);
   RUN_TEST(test_the_config_topic_of_each_kind);
+  RUN_TEST(test_deleting_a_shade_clears_every_state_topic_the_broker_keeps);
+  RUN_TEST(test_deleting_a_group_clears_its_sun_sensor_topic);
   return UNITY_END();
 }

test/test_rollingcode/test_rollingcode.cpp

@@ -47,6 +47,16 @@ static void test_wraps_to_zero_after_0xFFFF() {
   TEST_ASSERT_EQUAL_UINT16(0, next(true, 0xFFFF, store));
 }
 
+// NVS is newer than shades.cfg, so a load keeps the higher code.
+static void test_a_load_keeps_the_higher_rolling_code() {
+  TEST_ASSERT_EQUAL_UINT16(500, rollingCodeAfterLoad(500, 300, false));
+  TEST_ASSERT_EQUAL_UINT16(700, rollingCodeAfterLoad(500, 700, false));
+}
+static void test_a_restore_takes_the_backups_rolling_code() {
+  TEST_ASSERT_EQUAL_UINT16(300, rollingCodeAfterLoad(500, 300, true));
+  TEST_ASSERT_EQUAL_UINT16(700, rollingCodeAfterLoad(500, 700, true));
+}
+
 int main(int argc, char **argv) {
   (void)argc;
   (void)argv;
@@ -55,5 +65,7 @@ int main(int argc, char **argv) {
   RUN_TEST(test_synced_never_reads_the_store);
   RUN_TEST(test_reading_does_not_use_up_a_code);
   RUN_TEST(test_wraps_to_zero_after_0xFFFF);
+  RUN_TEST(test_a_load_keeps_the_higher_rolling_code);
+  RUN_TEST(test_a_restore_takes_the_backups_rolling_code);
   return UNITY_END();
 }