Merge pull request #105 from cjkas/feat/proto-settings-ops

feat: settings ops, WiFi and frequency scans, reboot (protocol 2 step 1d)

.github/workflows/build.yaml

@@ -28,9 +28,26 @@ jobs:
       - name: Run native unit tests
         run: pio test -e native
 
+  webui:
+    name: Web UI tests
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+
+      - name: Set up Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+
+      # The protocol 2 client (data-src/js) against the recorded fixtures.
+      - name: Run web UI tests
+        run: node --test "test/js/*.test.js"
+
   build:
     # Don't build or publish firmware that fails its tests.
-    needs: test
+    needs: [test, webui]
     permissions: write-all
     name: ${{ matrix.name }}
     runs-on: ubuntu-latest

AGENTS.md

@@ -2,6 +2,7 @@
 - Fix any test or type errors until the whole suite is green.
 - Add or update tests for the code you change, even if nobody asked.
 - to test run `pio test -e native`
+- the web UI's protocol 2 client (`data-src/js/`): `node --test "test/js/*.test.js"`. The minify step joins those files into `index.js`, so each one ends with a `module.exports` footer for Node and adds nothing global but its classes.
 
 ## Building
 - Firmware builds need pioarduino's Core (`pip install pioarduino`, or the pioarduino IDE extension). Stock PlatformIO Core refuses the pinned platform, see [ADR 0003](docs/adr/0003-pioarduino-idf55.md).

data-src/index.html

@@ -8,9 +8,7 @@
     <meta name="apple-mobile-web-app-title" content="ESPSomfy RTS App">
     <meta name="apple-mobile-web-app-status-bar-style" content="black">
 
-    <link rel="stylesheet" href="main.css?v=3.0.14d" type="text/css" />
-    <link rel="stylesheet" href="widgets.css?v=3.0.14d" type="text/css" />
-    <link rel="stylesheet" href="icons.css?v=3.0.14d" type="text/css" />
+    <link rel="stylesheet" href="app.css?v=3.0.14d" type="text/css" />
     <link rel="icon" type="image/png" href="favicon.png" />
 
     <!-- iPad retina icon -->
@@ -181,10 +179,6 @@
                                 </select>
                                 <label for="fldPinEntry">Security Type</label>
                             </div>
-                            <div id="divPermissions" class="field-group" style="display:none;vertical-align:middle;float:right;width:auto;">
-                                <input id="cbSecureConfigOnly" data-bind="security.permissions.configOnly" type="checkbox" style="display:inline-block;" />
-                                <label for="cbSecureConfigOnly" style="display:inline-block;cursor:pointer;">Secure Config Only</label>
-                            </div>
                         </div>
                         <div id="divPinSecurity" style="display:none;">
                             <div class="field-group" style="text-align:center;">

data-src/js/client.js

@@ -0,0 +1,352 @@
+// A socket protocol 2 client (docs/PROTOCOL.md): ws://host/ws, hello, the sync
+// with next after each frame, ops answered by their reply, events and feeds.
+// It keeps the shade model (ProtoModel) up to date and connects again when the
+// socket closes.  The WebSocket, the timers and the schema's storage can be
+// handed in, so the tests (test/js) run it without a device.  The minify step
+// joins this file into index.js after schema.js and model.js.
+class ProtoError extends Error {
+    // code is an error name from the schema (badRequest, busy ...), or
+    // notConnected, closed or timeout from the client itself.
+    constructor(code, desc, data) {
+        super(desc || code);
+        this.code = code;
+        this.desc = desc || code;
+        this.data = data || {};
+    }
+}
+class ProtoClient {
+    // opts: host, token, WebSocket, storage (getItem/setItem for the schema;
+    // localStorage when there is one), setTimeout/clearTimeout, opTimeoutMs.
+    constructor(opts) {
+        const deps = ProtoClient.deps();
+        this.Schema = deps.ProtoSchema;
+        this.host = opts.host;
+        this.token = opts.token || '';
+        this.WebSocket = opts.WebSocket || (typeof WebSocket !== 'undefined' ? WebSocket : null);
+        this.storage = typeof opts.storage !== 'undefined' ? opts.storage : ProtoClient.browserStorage();
+        this.setTimeout = opts.setTimeout || ((fn, ms) => setTimeout(fn, ms));
+        this.clearTimeout = opts.clearTimeout || ((t) => clearTimeout(t));
+        this.opTimeoutMs = opts.opTimeoutMs || 10000;
+        this.schema = this.loadSchema() || new this.Schema();
+        this.model = new deps.ProtoModel(this.schema);
+        this.model.onChange((kind, id, item) => this.emit('change', { kind: kind, id: id, item: item }));
+        this.listeners = {};
+        this.pending = new Map();
+        this.feeds = new Set();
+        this.nextId = 1;
+        this.ws = null;
+        this.ready = false;        // synced: ops may go
+        this.schemaReady = false;  // frames can be read
+        this.helloHash = null;
+        this.opened = false;
+        this.stopped = true;
+        this.retries = 0;
+        this.reconnectTimer = null;
+        this.held = [];
+        this.fw = null;
+    }
+    // In the browser the classes are the ones index.js was joined with; in
+    // Node they are loaded.
+    static deps() {
+        if (typeof require === 'function' && typeof module !== 'undefined' && module.exports)
+            return { ProtoSchema: require('./schema.js').ProtoSchema, ProtoModel: require('./model.js').ProtoModel };
+        return { ProtoSchema: ProtoSchema, ProtoModel: ProtoModel };
+    }
+    static browserStorage() {
+        try {
+            return typeof localStorage !== 'undefined' ? localStorage : null;
+        } catch (err) {
+            return null;
+        }
+    }
+
+    // cb(d) for a device event by its name (shadeState, frame ...), and for
+    // the client's own: open, hello, synced, change, closed, versionMismatch,
+    // error (a refusal the device couldn't tie to an op) and refused.
+    // refused: the socket closed before it opened.  A browser can't tell a
+    // refused token from a device that is down, so the client tries again
+    // unless a listener calls close(), as the UI does once /loginContext says
+    // a login is needed (docs/SECURITY_PLAN.md).
+    on(name, cb) { (this.listeners[name] = this.listeners[name] || []).push(cb); }
+    off(name, cb) { this.listeners[name] = (this.listeners[name] || []).filter((x) => x !== cb); }
+    emit(name, d) { for (const cb of (this.listeners[name] || []).slice()) cb(d); }
+    // The next event of that name that match(d) accepts, or null after ms.
+    // An edit's events follow its reply, so this is how a caller gets the
+    // item the edit left: the model has it by then.
+    waitFor(name, match, ms) {
+        return new Promise((resolve) => {
+            let timer = null;
+            const cb = (d) => {
+                if (match && !match(d)) return;
+                this.off(name, cb);
+                this.clearTimeout(timer);
+                resolve(d);
+            };
+            this.on(name, cb);
+            timer = this.setTimeout(() => {
+                this.off(name, cb);
+                resolve(null);
+            }, ms);
+        });
+    }
+
+    connect() {
+        if (!this.stopped && (this.ws || this.reconnectTimer)) return;
+        this.stopped = false;
+        this.open();
+    }
+    close() {
+        this.stopped = true;
+        this.cancelReconnect();
+        if (this.ws) this.ws.close();
+    }
+    // A new token, from a login: the device checks it as the socket opens, so
+    // the client connects again with it at once.
+    setToken(token) {
+        this.token = token || '';
+        this.stopped = false;
+        this.retries = 0;
+        this.open();
+    }
+    url() {
+        const params = [];
+        if (this.token) params.push(`apikey=${encodeURIComponent(this.token)}`);
+        if (this.schema.hash) params.push(`schema=${encodeURIComponent(this.schema.hash)}`);
+        return `ws://${this.host}/ws${params.length ? '?' + params.join('&') : ''}`;
+    }
+    cancelReconnect() {
+        if (this.reconnectTimer) this.clearTimeout(this.reconnectTimer);
+        this.reconnectTimer = null;
+    }
+    // One socket at a time: one being replaced is let go without a word.
+    open() {
+        this.cancelReconnect();
+        if (this.ws) {
+            const old = this.ws;
+            this.ws = null;
+            this.lost();
+            old.close();
+        }
+        this.ready = false;
+        this.schemaReady = false;
+        this.opened = false;
+        this.held = [];
+        const ws = new this.WebSocket(this.url());
+        this.ws = ws;
+        ws.onopen = () => {
+            if (this.ws !== ws) return;
+            this.opened = true;
+            this.retries = 0;
+            this.emit('open');
+        };
+        ws.onmessage = (evt) => {
+            if (this.ws === ws) this.receive(evt.data);
+        };
+        ws.onclose = () => {
+            if (this.ws !== ws) return;
+            this.ws = null;
+            this.lost();
+            this.emit(this.opened ? 'closed' : 'refused');
+            if (this.stopped) return;
+            // Soon after a close, slower after each failure, at most 10 s.
+            const ms = Math.min(500 * Math.pow(2, this.retries++), 10000);
+            this.reconnectTimer = this.setTimeout(() => this.open(), ms);
+        };
+    }
+    // The socket has gone: nothing waiting gets its reply.
+    lost() {
+        this.ready = false;
+        for (const [, p] of this.pending) {
+            this.clearTimeout(p.timer);
+            p.reject(new ProtoError('closed', 'The connection closed'));
+        }
+        this.pending.clear();
+    }
+    frame(type, d, id) {
+        const frame = { v: ProtoClient.VERSION, t: this.schema.typeNumbers[type] };
+        if (typeof frame.t === 'undefined') throw new ProtoError('unknownOp', `The device has no message ${type}`);
+        if (id) frame.id = id;
+        if (d) frame.d = d;
+        return JSON.stringify(frame);
+    }
+
+    // Sends an op, or a control message that wants a reply, by its name with
+    // its fields by name.  Resolves with the reply's fields, rejects with a
+    // ProtoError.
+    op(name, fields) {
+        return new Promise((resolve, reject) => {
+            if (!this.ready || !this.ws) {
+                reject(new ProtoError('notConnected', 'Not connected to the device'));
+                return;
+            }
+            const id = this.nextId++;
+            let text;
+            try {
+                text = this.frame(name, this.schema.pack(fields || {}), id);
+            } catch (err) {
+                reject(err instanceof ProtoError ? err : new ProtoError('badRequest', err.message));
+                return;
+            }
+            this.ws.send(text);
+            const timer = this.setTimeout(() => {
+                this.pending.delete(id);
+                reject(new ProtoError('timeout', `No reply to ${name}`));
+            }, this.opTimeoutMs);
+            this.pending.set(id, { resolve: resolve, reject: reject, timer: timer, text: text, resends: 0 });
+        });
+    }
+    // Feeds stay subscribed across reconnects.
+    subscribe(feed) {
+        if (typeof this.schema.feedNumbers[feed] === 'undefined') return Promise.reject(new ProtoError('badRequest', `The device has no feed ${feed}`));
+        this.feeds.add(feed);
+        return this.op('subscribe', { feed: this.schema.feedNumbers[feed] });
+    }
+    unsubscribe(feed) {
+        this.feeds.delete(feed);
+        if (typeof this.schema.feedNumbers[feed] === 'undefined') return Promise.resolve();
+        return this.op('unsubscribe', { feed: this.schema.feedNumbers[feed] });
+    }
+    // The shade model and status again, without the schema.
+    resync() {
+        this.model.beginSync();
+        return this.op('sync').catch((err) => {
+            this.model.cancelSync();
+            throw err;
+        });
+    }
+
+    receive(text) {
+        let frame;
+        try {
+            frame = JSON.parse(text);
+        } catch (err) {
+            return;
+        }
+        if (frame.v !== ProtoClient.VERSION) {
+            this.emit('versionMismatch', { device: frame.v, client: ProtoClient.VERSION });
+            return;
+        }
+        const d = frame.d || {};
+        // These have fixed numbers: they come before the schema.
+        switch (frame.t) {
+            case ProtoClient.HELLO: this.hello(d); return;
+            case ProtoClient.SYNC: this.syncFrame(d); return;
+            case ProtoClient.SYNC_END: this.syncEnd(); return;
+            default: break;
+        }
+        // A frame the client can't read yet waits for the schema.
+        if (!this.schemaReady) this.held.push(frame);
+        else this.dispatch(frame);
+    }
+    hello(d) {
+        this.fw = d.fw;
+        this.helloHash = d.sh;
+        // A schema the client kept for this one needn't come again; any other
+        // is no use, and the device sends the one it has.
+        if (this.schema.hash !== d.sh) {
+            this.schema = new this.Schema();
+            this.model.schema = this.schema;
+        }
+        this.model.beginSync();
+        this.emit('hello', d);
+        if (this.schema.hash === d.sh) this.readable();
+    }
+    syncFrame(d) {
+        this.schema.add(d);
+        const device = this.Schema.withoutRows(d);
+        // The schema's rows come first, so a frame with the device's sections
+        // has the whole of it.
+        if (Object.keys(device).length) {
+            this.readable();
+            this.model.applySync(this.schema.expand(device));
+        }
+        this.ws.send(JSON.stringify({ v: ProtoClient.VERSION, t: ProtoClient.NEXT }));
+    }
+    // The schema is whole: frames that waited for it go, in the order they
+    // came and ahead of the sync frames formatted after them.
+    readable() {
+        if (this.schemaReady) return;
+        this.schemaReady = true;
+        const held = this.held;
+        this.held = [];
+        for (const frame of held) this.dispatch(frame);
+    }
+    syncEnd() {
+        this.readable();
+        this.model.endSync();
+        if (!this.ready) {
+            this.ready = true;
+            // Only a whole schema is kept, so a sync cut short gets it again.
+            this.schema.hash = this.helloHash;
+            this.saveSchema();
+            for (const feed of this.feeds) this.op('subscribe', { feed: this.schema.feedNumbers[feed] }).catch(() => {});
+        }
+        this.emit('synced', this.model);
+    }
+    dispatch(frame) {
+        const d = this.schema.expand(frame.d || {});
+        if (frame.t === ProtoClient.REPLY) this.reply(frame.id, d);
+        else {
+            const name = this.schema.types[frame.t];
+            if (!name) return;
+            this.model.applyEvent(name, d);
+            this.emit(name, d);
+        }
+    }
+    reply(id, d) {
+        const err = d.ok ? null : new ProtoError(this.schema.errors[d.code] || 'internal', d.desc, d);
+        const p = typeof id !== 'undefined' ? this.pending.get(id) : null;
+        if (p) {
+            this.pending.delete(id);
+            this.clearTimeout(p.timer);
+            if (err) p.reject(err);
+            else p.resolve(d);
+            return;
+        }
+        if (!err) return;
+        // A long message the device couldn't take while it put another one
+        // together is refused without its id: the long ones waiting go again.
+        if (err.code === 'busy' && this.resendLong()) return;
+        this.emit('error', err);
+    }
+    resendLong() {
+        let any = false;
+        for (const [, p] of this.pending) {
+            if (p.text.length < ProtoClient.LONG_MESSAGE || p.resends >= 3) continue;
+            p.resends++;
+            any = true;
+            this.setTimeout(() => {
+                if (this.ws && [...this.pending.values()].includes(p)) this.ws.send(p.text);
+            }, 200 * p.resends);
+        }
+        return any;
+    }
+
+    loadSchema() {
+        if (!this.storage) return null;
+        try {
+            return this.Schema.load(JSON.parse(this.storage.getItem(ProtoClient.SCHEMA_KEY)));
+        } catch (err) {
+            return null;
+        }
+    }
+    saveSchema() {
+        if (!this.storage) return;
+        try {
+            this.storage.setItem(ProtoClient.SCHEMA_KEY, JSON.stringify(this.schema.save()));
+        } catch (err) { /* Private windows may have no storage: the device sends the schema each time. */ }
+    }
+}
+ProtoClient.VERSION = 2;
+// The types a client reads before it has the schema, and next, which it sends
+// during the sync.
+ProtoClient.HELLO = 1;
+ProtoClient.REPLY = 2;
+ProtoClient.NEXT = 3;
+ProtoClient.SYNC = 4;
+ProtoClient.SYNC_END = 5;
+// A message longer than a TCP segment reaches the device in pieces
+// (docs/PROTOCOL.md).
+ProtoClient.LONG_MESSAGE = 1400;
+ProtoClient.SCHEMA_KEY = 'espsomfyProtoSchema';
+if (typeof module !== 'undefined' && module.exports) module.exports = { ProtoClient, ProtoError };

data-src/js/model.js

@@ -0,0 +1,159 @@
+// The shade model as socket protocol 2 keeps a client up to date with it
+// (docs/PROTOCOL.md): rooms, shades, groups and repeaters from the sync, then
+// changed by the device's events.  Items are inserted or updated by id, so an
+// event during the sync does no harm.  It takes frames already expanded to
+// names (ProtoSchema.expand) and fills in what a frame leaves out at its
+// default.  The minify step joins this file into index.js; Node loads it on
+// its own for the tests (test/js).
+class ProtoModel {
+    constructor(schema) {
+        this.schema = schema;
+        this.rooms = new Map();
+        this.shades = new Map();
+        this.groups = new Map();
+        this.repeaters = [];
+        this.header = {};
+        this.status = {};
+        this.listeners = [];
+        this.seen = null;
+    }
+    // cb(kind, id, item): kind is room, shade or group for an item added or
+    // changed; roomRemoved, shadeRemoved or groupRemoved; roomOrder,
+    // shadeOrder or groupOrder (item: the ids in order); repeaters, header or
+    // status.
+    onChange(cb) { this.listeners.push(cb); }
+    changed(kind, id, item) { for (const cb of this.listeners) cb(kind, id, item); }
+
+    // A sync: what it doesn't list is gone once it ends, the repeaters too,
+    // which it leaves out when there are none.
+    beginSync() { this.seen = { rooms: new Set(), shades: new Set(), groups: new Set(), repeaters: false }; }
+    // A sync that won't come after all.
+    cancelSync() { this.seen = null; }
+    applySync(d) {
+        if (d.header) {
+            this.header = this.schema.fill(Object.assign({}, d.header), ProtoModel.FIELDS.header);
+            this.changed('header', 0, this.header);
+        }
+        for (const kind of ProtoModel.ITEMS) for (const item of d[kind.list] || []) this.put(kind, item);
+        if (d.repeaters) this.putRepeaters(d.repeaters);
+        if (d.status) {
+            this.putStatus(d.status, ProtoModel.FIELDS.memory);
+            this.putStatus(d.status, this.header.connType === 'Ethernet' ? ProtoModel.FIELDS.ethernet : ProtoModel.FIELDS.wifi);
+        }
+    }
+    endSync() {
+        if (!this.seen) return;
+        for (const kind of ProtoModel.ITEMS) {
+            const map = this[kind.list];
+            for (const id of [...map.keys()]) {
+                if (this.seen[kind.list].has(id)) continue;
+                map.delete(id);
+                this.changed(kind.removed, id);
+            }
+        }
+        if (!this.seen.repeaters && this.repeaters.length) this.putRepeaters([]);
+        this.seen = null;
+    }
+    // An event by its type's name; false for one that isn't about the model,
+    // or that changed nothing.
+    applyEvent(type, d) {
+        for (const kind of ProtoModel.ITEMS) {
+            const id = d[kind.id] || 0;
+            switch (type) {
+                case kind.name: this.put(kind, d); return true;
+                case kind.state: return this.mergeState(kind, id, d);
+                case kind.removed: return this.remove(kind, id);
+                case kind.order: return this.order(kind, d.ids || []);
+                default: break;
+            }
+        }
+        switch (type) {
+            case 'repeaters': this.putRepeaters(d.repeaters || []); return true;
+            case 'memory': this.putStatus(d, ProtoModel.FIELDS.memory); return true;
+            case 'wifi': this.putStatus(d, ProtoModel.FIELDS.wifi); return true;
+            case 'ethernet': this.putStatus(d, ProtoModel.FIELDS.ethernet); return true;
+            default: return false;
+        }
+    }
+
+    // The lists in their sort order.
+    roomList() { return ProtoModel.sorted(this.rooms); }
+    shadeList() { return ProtoModel.sorted(this.shades); }
+    groupList() { return ProtoModel.sorted(this.groups); }
+    static sorted(map) { return [...map.values()].sort((a, b) => a.sortOrder - b.sortOrder); }
+
+    // A whole item: what it had before is replaced.
+    put(kind, item) {
+        const id = item[kind.id] || 0;
+        const whole = this.schema.fill(Object.assign({}, item), ProtoModel.FIELDS[kind.name]);
+        // A shade's linked remotes leave out their defaults too.
+        if (kind.name === 'shade')
+            whole.linkedRemotes = whole.linkedRemotes.map((r) => this.schema.fill(Object.assign({}, r), ProtoModel.FIELDS.linkedRemote));
+        this[kind.list].set(id, whole);
+        if (this.seen) this.seen[kind.list].add(id);
+        this.changed(kind.name, id, whole);
+    }
+    // Part of an item: only its state.  A state of an item not synced yet
+    // waits for the item.
+    mergeState(kind, id, d) {
+        const item = this[kind.list].get(id);
+        if (!item) return false;
+        Object.assign(item, this.schema.fill(Object.assign({}, d), ProtoModel.FIELDS[kind.state]));
+        this.changed(kind.name, id, item);
+        return true;
+    }
+    remove(kind, id) {
+        if (!this[kind.list].delete(id)) return false;
+        this.changed(kind.removed, id);
+        return true;
+    }
+    order(kind, ids) {
+        ids.forEach((id, i) => {
+            const item = this[kind.list].get(id);
+            if (item) item.sortOrder = i;
+        });
+        this.changed(kind.order, 0, ids);
+        return true;
+    }
+    putRepeaters(list) {
+        this.repeaters = list.slice();
+        if (this.seen) this.seen.repeaters = true;
+        this.changed('repeaters', 0, this.repeaters);
+    }
+    putStatus(d, fields) {
+        const part = {};
+        for (const name of fields) part[name] = typeof d[name] === 'undefined' ? this.schema.defaultOf(name) : d[name];
+        Object.assign(this.status, part);
+        this.changed('status', 0, this.status);
+    }
+}
+// Rooms, shades and groups: their list, id, and the events about them.  Rooms
+// have no state of their own.
+ProtoModel.ITEMS = [
+    { name: 'room', list: 'rooms', id: 'roomId', state: null, removed: 'roomRemoved', order: 'roomOrder' },
+    { name: 'shade', list: 'shades', id: 'shadeId', state: 'shadeState', removed: 'shadeRemoved', order: 'shadeOrder' },
+    { name: 'group', list: 'groups', id: 'groupId', state: 'groupState', removed: 'groupRemoved', order: 'groupOrder' }
+];
+// The fields of each kind of item, as the device writes them
+// (src/ProtoSync.cpp, src/outputs/ProtoSocket.cpp): what a frame that leaves
+// one out means by it.  The schema has each field's default, not which items
+// have it.
+ProtoModel.FIELDS = {
+    header: ['maxRooms', 'maxShades', 'maxGroups', 'maxGroupedShades', 'maxLinkedRemotes', 'startingAddress', 'bitLength',
+        'proto', 'serverId', 'hostname', 'model', 'chipModel', 'fw', 'connType'],
+    room: ['roomId', 'name', 'sortOrder'],
+    shadeState: ['shadeId', 'position', 'target', 'direction', 'myPos', 'tiltPosition', 'tiltTarget', 'tiltDirection',
+        'myTiltPos', 'flags', 'lastRollingCode'],
+    groupState: ['groupId', 'flags', 'direction'],
+    linkedRemote: ['remoteAddress', 'lastRollingCode'],
+    group: ['groupId', 'roomId', 'name', 'remoteAddress', 'lastRollingCode', 'bitLength', 'proto', 'sunSensor',
+        'flipCommands', 'flags', 'repeats', 'sortOrder', 'linkedShades'],
+    memory: ['freeHeap', 'maxHeap', 'minHeap', 'totalHeap', 'uptime', 'loopRate', 'loopMaxUs'],
+    wifi: ['ssid', 'strength', 'channel'],
+    ethernet: ['connected', 'speed', 'fullDuplex']
+};
+ProtoModel.FIELDS.shade = ProtoModel.FIELDS.shadeState.concat(['roomId', 'name', 'remoteAddress', 'upTime', 'downTime',
+    'tiltTime', 'stepSize', 'paired', 'tiltType', 'shadeType', 'bitLength', 'proto', 'flipCommands', 'flipPosition',
+    'inGroup', 'sunSensor', 'light', 'simMy', 'repeats', 'sortOrder', 'gpioUp', 'gpioDown', 'gpioMy', 'gpioLLTrigger',
+    'linkedRemotes']);
+if (typeof module !== 'undefined' && module.exports) module.exports = { ProtoModel };

data-src/js/schema.js

@@ -0,0 +1,101 @@
+// Socket protocol 2's schema (docs/PROTOCOL.md): the device's table of message
+// types, fields, error codes and feeds, as the sync sends it.  It turns the
+// short keys of a frame into names and back.  The minify step joins this file
+// into index.js; Node loads it on its own for the tests (test/js).
+class ProtoSchema {
+    constructor() {
+        this.hash = null;
+        this.types = {};        // number -> name
+        this.typeNumbers = {};  // name -> number
+        this.fields = {};       // key -> { name, kind, def }
+        this.keys = {};         // name -> key
+        this.errors = {};
+        this.errorNumbers = {};
+        this.feedNumbers = {};
+        this.rows = { T: [], F: [], E: [], Q: [] };
+    }
+    // A sync frame's payload without the schema's rows: the device's sections.
+    static withoutRows(d) {
+        const out = {};
+        for (const key of Object.keys(d)) if (!ProtoSchema.ROWS.includes(key)) out[key] = d[key];
+        return out;
+    }
+    // The schema's rows in a sync frame's payload: T types, F fields, E error
+    // codes, Q feeds.  They may span several frames.
+    add(d) {
+        for (const [number, name] of d.T || []) {
+            this.types[number] = name;
+            this.typeNumbers[name] = number;
+        }
+        for (const [key, name, kind, def] of d.F || []) {
+            this.fields[key] = { name: name, kind: kind, def: def };
+            this.keys[name] = key;
+        }
+        for (const [number, name] of d.E || []) {
+            this.errors[number] = name;
+            this.errorNumbers[name] = number;
+        }
+        for (const [number, name] of d.Q || []) this.feedNumbers[name] = number;
+        for (const k of ProtoSchema.ROWS) if (d[k]) this.rows[k].push(...d[k]);
+    }
+    get empty() { return Object.keys(this.fields).length === 0; }
+    // What a field left out of a frame is: its default.
+    defaultOf(name) {
+        const f = this.fields[this.keys[name]];
+        if (!f) return undefined;
+        switch (f.kind) {
+            case 'b': return f.def !== 0;
+            case 's': return '';
+            case 'l': return [];
+            case 'o': return {};
+            default: return f.def;
+        }
+    }
+    // Short keys to names, and 1/0 to true/false, all the way down.  A field
+    // left out stays out; fill() puts the defaults in.
+    expand(value) {
+        if (Array.isArray(value)) return value.map((v) => this.expand(v));
+        if (value === null || typeof value !== 'object') return value;
+        const out = {};
+        for (const key of Object.keys(value)) {
+            const f = this.fields[key];
+            let v = value[key];
+            if (f && f.kind === 'b' && typeof v === 'number') v = v !== 0;
+            out[f ? f.name : key] = this.expand(v);
+        }
+        return out;
+    }
+    // Names to short keys, and booleans to 1/0, for an op's payload.  A name
+    // the schema doesn't have is an error: the device would refuse it anyway.
+    pack(value) {
+        if (Array.isArray(value)) return value.map((v) => this.pack(v));
+        if (value === null || typeof value !== 'object') return typeof value === 'boolean' ? (value ? 1 : 0) : value;
+        const out = {};
+        for (const name of Object.keys(value)) {
+            if (typeof value[name] === 'undefined') continue;
+            const key = this.keys[name];
+            if (typeof key === 'undefined') throw new Error(`The device has no field ${name}`);
+            out[key] = this.pack(value[name]);
+        }
+        return out;
+    }
+    // Sets each of names that obj lacks to its default, as a frame leaves out
+    // a field at its default.  Returns obj.
+    fill(obj, names) {
+        for (const name of names) if (typeof obj[name] === 'undefined') obj[name] = this.defaultOf(name);
+        return obj;
+    }
+    // What a cache keeps, and the schema back from it.
+    save() { return { hash: this.hash, rows: this.rows }; }
+    static load(saved) {
+        if (!saved || typeof saved.hash !== 'string' || !saved.rows) return null;
+        const s = new ProtoSchema();
+        s.add(saved.rows);
+        s.hash = saved.hash;
+        return s.empty ? null : s;
+    }
+}
+// The keys of the schema's rows in a sync frame, fixed as a client needs them
+// before it has the schema.
+ProtoSchema.ROWS = ['T', 'F', 'E', 'Q'];
+if (typeof module !== 'undefined' && module.exports) module.exports = { ProtoSchema };

data-src/js/views.js

@@ -0,0 +1,80 @@
+// What the web UI's panels bind, made from socket protocol 2's items, events
+// and settings (docs/PROTOCOL.md), and back.  The UI kept its names where they
+// differ from the protocol's: a frame's address and rcode, a scan's RSSI, the
+// network panel's wifi, ethernet and ip objects, the radio's type.  The minify
+// step joins this file into index.js; Node loads it on its own for the tests
+// (test/js).
+class ProtoViews {
+    // A settings section's reply with what the device left out at its default
+    // filled in, so a form shows what is saved rather than what it showed
+    // before.
+    static settings(schema, section, d) {
+        return schema.fill(Object.assign({}, d), ProtoViews.SETTINGS[section]);
+    }
+    // A group with its shades whole, where the device sends their ids.
+    static group(model, group) {
+        if (!group) return group;
+        const view = Object.assign({}, group);
+        view.linkedShades = group.linkedShades.map((id) => model.shades.get(id) || { shadeId: id, name: `Shade ${id}`, remoteAddress: 0 });
+        return view;
+    }
+    // A received frame as the frames list and remote linking read it.
+    static frame(schema, d) {
+        const f = schema.fill(Object.assign({}, d), ['encKey', 'remoteAddress', 'lastRollingCode', 'command', 'rawCmd', 'rssi', 'bits', 'proto',
+            'valid', 'pulses']);
+        return { encKey: f.encKey, address: f.remoteAddress, rcode: f.lastRollingCode, command: f.command, rawCmd: f.rawCmd, rssi: f.rssi,
+            bits: f.bits, proto: f.proto, valid: f.valid, stepSize: d.stepSize, pulses: f.pulses };
+    }
+    static scan(schema, d) {
+        const s = schema.fill(Object.assign({}, d), ['scanning', 'testFreq', 'testRssi', 'frequency', 'rssi']);
+        return { scanning: s.scanning, testFreq: s.testFreq, testRSSI: s.testRssi, frequency: s.frequency, RSSI: s.rssi };
+    }
+    // The network section as the panel binds it: WiFi, Ethernet and the
+    // address each in an object of their own, the connection as connType.
+    static network(schema, d) {
+        const n = ProtoViews.settings(schema, 'network', d);
+        const view = { connType: n.connMode, hasEthernet: n.hasEthernet, wifi: {}, ethernet: {}, ip: {} };
+        for (const name of ProtoViews.WIFI) view.wifi[name] = n[name];
+        for (const name of ProtoViews.ETHERNET) view.ethernet[name] = n[name];
+        for (const name of ProtoViews.IP) view.ip[name] = n[name];
+        return view;
+    }
+    // The radio section as the panel binds it, where the bit length is the
+    // radio's type.
+    static radio(schema, d) {
+        const config = ProtoViews.settings(schema, 'radio', d);
+        config.type = config.bitLength;
+        delete config.bitLength;
+        return config;
+    }
+    // A save of the radio panel's config: everything but what is only shown.
+    static radioSave(config) {
+        const fields = { section: 'radio', bitLength: config.type };
+        for (const name of ProtoViews.SETTINGS.radio) {
+            if (name !== 'bitLength' && name !== 'radioInit') fields[name] = config[name];
+        }
+        return fields;
+    }
+    // An op's refusal as the UI's error box shows it.  A shade whose settings
+    // don't add up carries the reason's number, which the errors table
+    // describes.
+    static opError(name, err) {
+        const e = { htmlError: err.code, service: name, desc: err.desc };
+        if (err.data && typeof err.data.validation === 'number' && err.data.validation !== 0) e.code = err.data.validation;
+        return e;
+    }
+}
+ProtoViews.WIFI = ['ssid', 'hasPassphrase', 'roaming', 'hidden'];
+ProtoViews.ETHERNET = ['boardType', 'phyType', 'CLKMode', 'phyAddress', 'PWRPin', 'MDCPin', 'MDIOPin'];
+ProtoViews.IP = ['dhcp', 'ip', 'gateway', 'subnet', 'dns1', 'dns2'];
+// The fields of each settings section, as getSettings sends them
+// (src/DeviceOps.cpp).
+ProtoViews.SETTINGS = {
+    general: ['hostname', 'ssdpBroadcast', 'ntpServer', 'posixZone'],
+    network: ['connMode', 'hasEthernet'].concat(ProtoViews.WIFI, ProtoViews.ETHERNET, ProtoViews.IP),
+    mqtt: ['enabled', 'pubDisco', 'protocol', 'hostname', 'port', 'username', 'hasPassword', 'rootTopic', 'discoTopic'],
+    security: ['securityType', 'username', 'hasPassword', 'hasPin'],
+    radio: ['bitLength', 'proto', 'enabled', 'TXPin', 'RXPin', 'SCKPin', 'MOSIPin', 'MISOPin', 'CSNPin', 'frequency', 'rxBandwidth',
+        'deviation', 'txPower', 'noiseDetection', 'radioInit']
+};
+if (typeof module !== 'undefined' && module.exports) module.exports = { ProtoViews };

data-src/login.html

@@ -1,53 +0,0 @@
-<!DOCTYPE html>
-<html>
-<head>
-    <meta name="viewport" content="width=device-width, initial-scale=1">
-    <meta charset="UTF-8">
-    <link rel="stylesheet" href="main.css?v=2.4.0" type="text/css" />
-    <link rel="stylesheet" href="widgets.css?v=2.4.0" type="text/css" />
-    <link rel="stylesheet" href="icons.css?v=2.4.0" type="text/css" />
-    <link rel="icon" type="image/png" href="favicon.png" />
-    <script type="text/javascript" src="index.js?v=2.4.0"></script>
-</head>
-<body onload="general.loadLogin();">
-    <div id="divContainer" class="container" data-securitytype="0">
-        <h1 style="text-align: center;"><img src="icon.png" style="width:127px;margin-left:1px;margin-top:-10px;" /></h1>
-        <div id="divLoginPnl" class="login-content" style="position:relative;">
-            <div style="max-width:270px;margin:0px auto;">
-                <form id="frmLogin" action="/login" method="post" class="login-form" onsubmit="general.login(event); return false;">
-                    <input id="fldPin" type="hidden" name="pin">
-                    <div id="divPinSecurity" style="display:none;">
-                        <div class="field-group" style="text-align:center;">
-                            <div id="fldPinEntry" style="display:inline-block;" onkeydown="general.pinKeyPressed(event);">
-                                <input class="pin-digit" maxlength="1" data-bind="security.pin.d0" onfocus="general.pinDigitFocus(event);">
-                                <input class="pin-digit" maxlength="1" data-bind="security.pin.d1" onfocus="general.pinDigitFocus(event);">
-                                <input class="pin-digit" maxlength="1" data-bind="security.pin.d2" onfocus="general.pinDigitFocus(event);">
-                                <input class="pin-digit" maxlength="1" data-bind="security.pin.d3" onfocus="general.pinDigitFocus(event);">
-                            </div>
-                            <label for="fldPinEntry" style="margin-top:7px;">Enter Pin</label>
-                        </div>
-                    </div>
-                    <div id="divPasswordSecurity" style="display:none;">
-                        <div class="field-group">
-                            <input id="fldUsername" name="username" type="text" autocomplete="username" data-bind="security.username" length=32 placeholder="Username">
-                            <label for="fldUsername">Username</label>
-                        </div>
-                        <div class="field-group">
-                            <input id="fldPassword" name="password" type="password" autocomplete="current-password" data-bind="security.password" length=32 placeholder="Password">
-                            <label for="fldPassword">Password</label>
-                        </div>
-                    </div>
-                    <div style="text-align:center;"><span id="spanLoginMessage" style="color:red"></span></div>
-                    <div class="button-container">
-                        <button id="btnLogin" type="submit" value="Submit">
-                            Login
-                        </button>
-                    </div>
-                </form>
-            </div>
-        </div>
-    </div>
-    <script type="text/javascript">
-    </script>
-</body>
-</html>

data-src/main.css

@@ -868,3 +868,14 @@ div.frame-pulses {
         padding-right:14px;
     }
 }
+/* Shown when the device speaks another protocol version than this page. */
+.version-banner {
+    position: sticky;
+    top: 0;
+    z-index: 100;
+    padding: 8px 12px;
+    background: orangered;
+    color: white;
+    font-size: 14px;
+    text-align: center;
+}

docs/PROTOCOL.md

@@ -39,8 +39,8 @@ The keys of `hello` (`fw`, `sh`) and of the schema itself (`T`, `F`, `E`, `Q`) n
 | Range | Sender | Types |
 |---|---|---|
 | 1–15 | control | `hello` 1, `reply` 2, `next` 3, `sync` 4, `syncEnd` 5, `subscribe` 6, `unsubscribe` 7 |
-| 16–127 | client ops | commands: `shadeCommand` 16, `groupCommand` 17, `tiltCommand` 18, `repeatCommand` 19, `setPositions` 20, `setSensor` 21, `setMyPosition` 22, `unpairShade` 23, `setRollingCode` 24, `sendRemoteCommand` 25; edits: 26–44 (`addRoom` … `groupSortOrder`); reads: `getNextRoom` 45, `getNextShade` 46, `getNextGroup` 47, `groupOptions` 48; settings follow |
-| 128–255 | device events | `shadeState`, `shade`, `shadeRemoved`, `group`, `groupState`, `groupRemoved`, `room`, `roomRemoved`, `roomOrder`, `shadeOrder`, `groupOrder`, `shadeCommanded`, `memory`, `wifi`, `ethernet`, `frame`, `frequencyScan`, `repeaters` |
+| 16–127 | client ops | commands: `shadeCommand` 16, `groupCommand` 17, `tiltCommand` 18, `repeatCommand` 19, `setPositions` 20, `setSensor` 21, `setMyPosition` 22, `unpairShade` 23, `setRollingCode` 24, `sendRemoteCommand` 25; edits: 26–44 (`addRoom` … `groupSortOrder`); reads: `getNextRoom` 45, `getNextShade` 46, `getNextGroup` 47, `groupOptions` 48; settings and the device: `getSettings` 49, `saveSettings` 50, `scanWifi` 51, `beginFrequencyScan` 52, `endFrequencyScan` 53, `reboot` 54 |
+| 128–255 | device events | `shadeState`, `shade`, `shadeRemoved`, `group`, `groupState`, `groupRemoved`, `room`, `roomRemoved`, `roomOrder`, `shadeOrder`, `groupOrder`, `shadeCommanded`, `memory`, `wifi`, `ethernet`, `frame`, `frequencyScan`, `repeaters`, `settingsChanged`, `wifiNetworks` |
 
 A number is never reused. A retired type stays reserved.
 
@@ -73,6 +73,8 @@ Events go out to every client from the moment it connects, so they can arrive du
 | `repeaters` | a repeater was linked or unlinked | the list |
 | `shadeCommanded` | a remote or the device commanded a shade | id, address, command, source, rolling code, source address |
 | `memory`, `wifi`, `ethernet` | every 30 s, WiFi only when the signal moved | |
+| `settingsChanged` | another client saved settings | `section` |
+| `wifiNetworks` | a WiFi scan this client asked for is done; to that client only | `networks`, each with `ssid`, `channel`, `strength`, `macAddress` |
 
 ## Commands
 
@@ -124,6 +126,48 @@ What an edit form needs before it is sent, worked out by the device as the edit
 
 When every slot is taken the reply is a bad request; `groupOptions` for a group that doesn't exist is not found.
 
+## Settings
+
+The settings are in five sections. `getSettings` and `saveSettings` name one in `section`:
+
+| Section | Fields | The routes it replaces |
+|---|---|---|
+| `general` | `hostname`, `ssdpBroadcast`, `ntpServer`, `posixZone` | `/modulesettings`, `/setgeneral` |
+| `network` | `connMode`; WiFi: `ssid`, `passphrase`, `roaming`, `hidden`; Ethernet: `boardType`, `phyType`, `CLKMode`, `phyAddress`, `PWRPin`, `MDCPin`, `MDIOPin`; the address: `dhcp`, `ip`, `gateway`, `subnet`, `dns1`, `dns2` | `/networksettings`, `/setNetwork`, `/setIP`, `/connectwifi` |
+| `mqtt` | `enabled`, `pubDisco`, `protocol`, `hostname` (the broker's), `port`, `username`, `password`, `rootTopic`, `discoTopic` | `/mqttsettings`, `/connectmqtt` |
+| `security` | `securityType`, `username`, `password`, `pin` | `/getSecurity`, `/saveSecurity` |
+| `radio` | `bitLength`, `proto`, `enabled`, `TXPin`, `RXPin`, `SCKPin`, `MOSIPin`, `MISOPin`, `CSNPin`, `frequency`, `rxBandwidth`, `deviation`, `txPower`, `noiseDetection` | `/getRadio`, `/saveRadio` |
+
+- **Reading.** `getSettings` replies with the section's fields. A secret is never sent: `hasPassphrase`, `hasPassword` and `hasPin` say whether one is set. `network` also has `hasEthernet`, and has the Ethernet fields only on a device with Ethernet. `radio` also has `radioInit`.
+- `connMode` is the connection the settings ask for: 1 WiFi, 2 Ethernet, 3 Ethernet with WiFi as the fallback. The header's `connType` is the connection in use.
+- **Saving.** `saveSettings` takes the section and the fields to change. As with every op, a field left out stays as it is. A secret sent empty keeps the stored one, except alongside a new `ssid` or MQTT `username`, as on the routes.
+- Every field is checked before anything is saved:
+  - its kind;
+  - its range: `port` 1–65535, `txPower` -30–12, `bitLength` 56 or 80, `proto` 0, 1, 2, 8 or 9, a pin 0–48 (`PWRPin` and `phyAddress` may also be -1, for none), `frequency` 300–928, and so on;
+  - a string's length, which is what the setting holds: `hostname` 31, `pin` 4, and so on;
+  - an address: four numbers 0–255, or `""` for none.
+
+  A bad request also comes for a field the section doesn't save (a `has*` field, `radioInit`, a field of another section, Ethernet on a device without it), and for a save with no fields.
+- One save runs at a time, whichever client sent it: while one is waiting, another gets busy.
+- The reply comes once the settings are saved and applied, as their route applies them. A saved time zone takes effect at once, where the route waited for a reboot.
+  - `rebooting` 1 means the device restarts a second later for them. That happens for a new `connMode`, for a new SSID or passphrase while on WiFi with nobody on the soft AP, and for Ethernet settings while on Ethernet.
+  - A security save's reply has the client's new token in `apiKey`, or none under None.
+- The other clients get `settingsChanged` with the section.
+- A security save that leaves the device Secured, or changes its mode, closes every client instead, the one that saved it too, once its reply has gone. Under Secured the tokens may no longer be good, and a client that connected under None had none. The saver connects again with the token from its reply; the others log in when theirs is refused. A save from None to None closes nobody.
+- Where a section differs from its routes:
+  - `/connectwifi` refused an SSID the device couldn't hear; the save doesn't check. The check stalled the device for a scan, and a hidden network can't be heard anyway. A client offers the networks from `scanWifi`.
+  - `/modulesettings` and `/networksettings` also sent `fwVersion` and `chipModel`, which are in the header (`fw`, `chipModel`). Their `connType` is `connMode` in `network`, and `/networksettings`' hostname and `ssdpBroadcast` are in `general`.
+  - `/scanaps` also sent the network the device is on: that is the `wifi` event, and `status` in the sync.
+
+| Op | Payload | Reply |
+|---|---|---|
+| `scanWifi` | | ok at once. When the scan is done, `wifiNetworks` goes to each client that asked since the last one: the named networks that can be heard, at most 30. A failed scan lists none. |
+| `beginFrequencyScan` | | ok, or radio unavailable when the radio is disabled. The radio sends nothing until the scan ends. Subscribe to the `frequencyScan` feed to follow it. |
+| `endFrequencyScan` | | ok. A scan nobody ends stops after 5 minutes. |
+| `reboot` | | ok, and the device restarts half a second later. |
+
+These ops run on `loop()` like the rest. `endFrequencyScan` goes ahead of whatever is waiting in the loop queue: an unpair waits there for the radio to send again, which only the scan's end allows.
+
 ## Replies
 
 A client message with an `id` gets exactly one `reply` with that `id`, to that client only, whether it is an op or a control message such as `next`: `ok` 1 on success, or an error `code` and a `desc` for people. Error codes are in the schema: bad request, unknown op, unauthorized, not found, busy, radio unavailable, internal.

docs/SECURITY_PLAN.md

@@ -37,6 +37,7 @@
 - **Handshake:** the socket is `ws://host/ws` on port 80. The token goes in `?apikey=<token>`, because browsers can't set headers on a websocket. `handleHandshake` checks it with the same comparison the HTTP routes use. `request->url()` is the path only, so the token doesn't reach the slow-request log.
 - **Rules:** under None anyone can connect. Under Secured the handshake needs a valid token, and a connection that has one may send every op and subscribe to every feed.
 - **Logging in with a socket already open:** the UI reconnects the socket with the token. There is no in-band upgrade message.
+- **Changing the security settings:** a `saveSettings` of the security section that leaves the device Secured, or changes its mode, closes every socket, since their tokens may no longer be good, and a socket opened under None had none. The client that saved is closed once its reply, which carries its new token, has gone, so no socket stays open on a check it never passed. The others log in when refused.
 - **Origin:** a handshake whose `Origin` header doesn't match `Host` is refused.
 
 ### CORS

docs/adr/0005-one-websocket-protocol.md

@@ -22,6 +22,7 @@ We decided that the web UI and the Home Assistant integration do everything exce
 - **The send queue stays at 32 messages per client.** A full queue closes the client, which reconnects and syncs again. Dropping messages would leave a shade showing the wrong position with no way to notice.
 - **A JSON envelope that a binary encoding can reuse.** Every frame is `{"v":2,"t":<type>,"id":<n>,"d":{...}}`, with `id` only on ops and replies. Message types and error codes are numbers that fit one byte and are never reused. A later binary encoding would put the same four fields in a fixed header, be asked for at the upgrade, and use binary websocket frames. The `42[...]` socket.io framing goes.
 - **Short keys, described by a schema the device sends.** Payload keys are one or two characters from one global dictionary. Booleans are 1/0, and fields at their default are left out. The schema lists every type, field (name, kind, default) and error code. The firmware's table is the only copy, and its writers can only emit declared fields. `hello` carries the schema's hash, and a client that has it cached reconnects with `?schema=<hash>` to skip it. Clients expand frames to full names, so client code stays readable.
+- **The web UI loads two files of its own: `index.js` and `app.css`.** Each file a page load fetches costs the device an open response, so the minify step joins the CSS files into `app.css`, and the protocol 2 client (`data-src/js/`) into `index.js`. Each client module ends with a `module.exports` footer, so Node can load it for the tests, and in the browser it adds nothing global but its classes.
 - **Feeds.** High-rate events (received frames, frequency scan) go only to clients that subscribed to that feed.
 - **A clean break, released as 4.0.0.** The firmware speaks only protocol 2. The new HA integration falls back to the old HTTP API and events when a device doesn't send `hello`; old integrations stop working against 4.0.0. The web UI checks `v` and shows a banner when the firmware and the web application don't match.
 - **Tests.** Native tests cover parsing, the op table, permissions, the sync stream and the largest item fitting in 4 KB. A Python e2e suite, run by hand against a device, checks the protocol and survival under a reconnect storm and a page-load replay, and records fixtures. The web UI's client layer is tested with `node --test`. The HA integration tests its protocol module with plain pytest and the integration with pytest-homeassistant-custom-component, against a fake device that replays the fixtures.

minify.py

@@ -52,6 +52,16 @@ MINIFY_AND_GZIP = {".html", ".htm", ".css", ".js", ".json", ".svg", ".xml"}
 # Extensions that optipng can optimise
 PNG_EXTENSIONS = {".png"}
 
+# Files joined into one, in this order, before they are minified: a page load
+# then opens fewer connections, each of which costs the device heap
+# (docs/adr/0005-one-websocket-protocol.md).  The protocol 2 client in js/
+# comes before index.js, which uses it.  A source listed here isn't copied on
+# its own.
+BUNDLES = {
+    "index.js": ["js/schema.js", "js/model.js", "js/client.js", "js/views.js", "index.js"],
+    "app.css": ["main.css", "widgets.css", "icons.css"],
+}
+
 # Everything else is copied verbatim
 # ──────────────────────────────────────────────
 
@@ -252,6 +262,39 @@ def minify_all():
     total_final = 0
     file_count = 0
 
+    def report(rel_path, stats):
+        nonlocal total_original, total_final, file_count
+        pct = (1 - stats["final"] / stats["original"]) * 100 if stats["original"] > 0 else 0
+        print(
+            f"  {rel_path:<40s} "
+            f"{stats['original']:>8,d} → {stats['final']:>8,d} B "
+            f"({pct:+.0f}%)  [{stats['action']}]"
+        )
+        total_original += stats["original"]
+        total_final += stats["final"]
+        file_count += 1
+
+    bundled = set()
+    # Inside data/, so it goes even when a bundle fails: left there, it would
+    # end up in the LittleFS image.
+    work_dir = os.path.join(dst_dir, ".bundles")
+    os.makedirs(work_dir, exist_ok=True)
+    try:
+        for name, parts in BUNDLES.items():
+            joined = os.path.join(work_dir, name)
+            with open(joined, "w", encoding="utf-8") as out:
+                for part in parts:
+                    # utf-8-sig drops a byte order mark: mid-file it would be
+                    # read as part of the next selector, and that rule lost.
+                    with open(os.path.join(src_dir, part), "r", encoding="utf-8-sig") as f:
+                        out.write(f.read())
+                    # A file without a final newline would run into the next.
+                    out.write("\n")
+                    bundled.add(os.path.normpath(part))
+            report(f"{name} ({len(parts)} files)", process_file(joined, os.path.join(dst_dir, name)))
+    finally:
+        shutil.rmtree(work_dir, ignore_errors=True)
+
     for root, dirs, files in os.walk(src_dir):
         for fname in sorted(files):
             # Skip hidden files and editor temp files
@@ -260,20 +303,11 @@ def minify_all():
 
             src_path = os.path.join(root, fname)
             rel_path = os.path.relpath(src_path, src_dir)
+            if os.path.normpath(rel_path) in bundled:
+                continue
             dst_path = os.path.join(dst_dir, rel_path)
 
-            stats = process_file(src_path, dst_path)
-
-            pct = (1 - stats["final"] / stats["original"]) * 100 if stats["original"] > 0 else 0
-            print(
-                f"  {rel_path:<40s} "
-                f"{stats['original']:>8,d} → {stats['final']:>8,d} B "
-                f"({pct:+.0f}%)  [{stats['action']}]"
-            )
-
-            total_original += stats["original"]
-            total_final += stats["final"]
-            file_count += 1
+            report(rel_path, process_file(src_path, dst_path))
 
     print(f"[minify] {file_count} files processed")
     print(

platformio.ini

@@ -108,7 +108,7 @@ build_flags =
 platform = native
 test_framework = unity
 test_build_src = yes
-build_src_filter = -<*> +<Utils.cpp> +<SomfyFrame.cpp> +<JsonFormatter.cpp> +<ShadeMotion.cpp> +<ShadeSensors.cpp> +<ShadeBehaviour.cpp> +<PulseDecoder.cpp> +<TxBuffer.cpp> +<RadioScheduler.cpp> +<ShadeRequests.cpp> +<entryPoints/MqttCommands.cpp> +<ModelSnapshot.cpp> +<ModelEditor.cpp> +<ShadeChanges.cpp> +<ProtoSchema.cpp> +<ProtoFrame.cpp> +<ProtoSync.cpp> +<ProtoParse.cpp> +<ProtoSlots.cpp> +<ProtoOps.cpp> +<ProtoEdits.cpp> +<ProtoAssembly.cpp>
+build_src_filter = -<*> +<Utils.cpp> +<SomfyFrame.cpp> +<JsonFormatter.cpp> +<ShadeMotion.cpp> +<ShadeSensors.cpp> +<ShadeBehaviour.cpp> +<PulseDecoder.cpp> +<TxBuffer.cpp> +<RadioScheduler.cpp> +<ShadeRequests.cpp> +<entryPoints/MqttCommands.cpp> +<ModelSnapshot.cpp> +<ModelEditor.cpp> +<ShadeChanges.cpp> +<ProtoSchema.cpp> +<ProtoFrame.cpp> +<ProtoSync.cpp> +<ProtoParse.cpp> +<ProtoSlots.cpp> +<ProtoOps.cpp> +<ProtoEdits.cpp> +<ProtoSettings.cpp> +<ProtoAssembly.cpp>
 lib_deps =
 	bblanchon/ArduinoJson@^7.2.2
 build_flags =

src/DeviceOps.cpp

@@ -0,0 +1,265 @@
+#include <Arduino.h>
+#include <WiFi.h>
+#include <atomic>
+#include <new>
+#include "esp_log.h"
+#include "DeviceOps.h"
+#include "ConfigSettings.h"
+#include "ESPNetwork.h"
+#include "MqttConnection.h"
+#include "Secrets.h"
+#include "Somfy.h"
+#include "Utils.h"
+#include "entryPoints/Web.h"
+#include "outputs/ProtoSocket.h"
+
+static const char *TAG = "DeviceOps";
+
+extern ConfigSettings settings;
+extern SomfyShadeController somfy;
+extern ESPNetwork net;
+extern MqttConnection mqtt;
+extern Web webServer;
+extern rebootDelay_t rebootDelay;
+
+static std::atomic<bool> saveBusy(false);
+
+settings_save_t *newSettingsSave(bool &busy) {
+  busy = saveBusy.exchange(true);
+  if(busy) return nullptr;
+  settings_save_t *save = new (std::nothrow) settings_save_t();
+  if(!save) saveBusy.store(false);
+  return save;
+}
+void freeSettingsSave(settings_save_t *save) {
+  delete save;
+  saveBusy.store(false);
+}
+
+/*********************************************************************
+ * getSettings: what the settings routes send, under the dictionary's
+ * keys.  Secrets only say whether they are set.
+ ********************************************************************/
+// An address of 0.0.0.0 is sent as "", as IPSettings::toJSON() does.
+static void addAddress(ProtoWriter &w, proto_field_t field, const IPAddress &ip) {
+  if(ip != IPAddress(0, 0, 0, 0)) w.add(field, ip.toString().c_str());
+}
+static void writeSettings(ProtoWriter &w, settings_section_t section) {
+  switch(section) {
+    case settings_section_t::general:
+      w.add(proto_field_t::hostname, settings.hostname);
+      w.add(proto_field_t::ssdpBroadcast, settings.ssdpBroadcast);
+      w.add(proto_field_t::ntpServer, settings.NTP.ntpServer);
+      w.add(proto_field_t::posixZone, settings.NTP.posixZone);
+      break;
+    case settings_section_t::network:
+      w.add(proto_field_t::connMode, static_cast<uint8_t>(settings.connType));
+      w.add(proto_field_t::ssid, settings.WIFI.ssid);
+      w.add(proto_field_t::hasPassphrase, hasSecret(settings.WIFI.passphrase));
+      w.add(proto_field_t::roaming, settings.WIFI.roaming);
+      w.add(proto_field_t::hidden, settings.WIFI.hidden);
+#if HAS_ETHERNET
+      w.add(proto_field_t::hasEthernet, true);
+      w.add(proto_field_t::boardType, settings.Ethernet.boardType);
+      w.add(proto_field_t::phyType, settings.Ethernet.phyType);
+      w.add(proto_field_t::CLKMode, static_cast<uint8_t>(settings.Ethernet.CLKMode));
+      w.add(proto_field_t::phyAddress, settings.Ethernet.phyAddress);
+      w.add(proto_field_t::PWRPin, settings.Ethernet.PWRPin);
+      w.add(proto_field_t::MDCPin, settings.Ethernet.MDCPin);
+      w.add(proto_field_t::MDIOPin, settings.Ethernet.MDIOPin);
+#endif
+      w.add(proto_field_t::dhcp, settings.IP.dhcp);
+      addAddress(w, proto_field_t::ip, settings.IP.ip);
+      addAddress(w, proto_field_t::gateway, settings.IP.gateway);
+      addAddress(w, proto_field_t::subnet, settings.IP.subnet);
+      addAddress(w, proto_field_t::dns1, settings.IP.dns1);
+      addAddress(w, proto_field_t::dns2, settings.IP.dns2);
+      break;
+    case settings_section_t::mqtt:
+      w.add(proto_field_t::enabled, settings.MQTT.enabled);
+      w.add(proto_field_t::pubDisco, settings.MQTT.pubDisco);
+      w.add(proto_field_t::protocol, settings.MQTT.protocol);
+      w.add(proto_field_t::hostname, settings.MQTT.hostname);
+      w.add(proto_field_t::port, settings.MQTT.port);
+      w.add(proto_field_t::username, settings.MQTT.username);
+      w.add(proto_field_t::hasPassword, hasSecret(settings.MQTT.password));
+      w.add(proto_field_t::rootTopic, settings.MQTT.rootTopic);
+      w.add(proto_field_t::discoTopic, settings.MQTT.discoTopic);
+      break;
+    case settings_section_t::security:
+      w.add(proto_field_t::securityType, static_cast<uint8_t>(settings.Security.type));
+      w.add(proto_field_t::username, settings.Security.username);
+      w.add(proto_field_t::hasPassword, hasSecret(settings.Security.password));
+      w.add(proto_field_t::hasPin, hasSecret(settings.Security.pin));
+      // Not "Secure Config Only"'s permissions: the socket has two modes, None
+      // and Secured (docs/adr/0006-two-security-modes-eleven-http-routes.md).
+      break;
+    case settings_section_t::radio: {
+      const transceiver_config_t &cfg = somfy.transceiver.config;
+      w.add(proto_field_t::bitLength, cfg.type);
+      w.add(proto_field_t::proto, static_cast<uint8_t>(cfg.proto));
+      w.add(proto_field_t::enabled, cfg.enabled);
+      w.add(proto_field_t::TXPin, cfg.TXPin);
+      w.add(proto_field_t::RXPin, cfg.RXPin);
+      w.add(proto_field_t::SCKPin, cfg.SCKPin);
+      w.add(proto_field_t::MOSIPin, cfg.MOSIPin);
+      w.add(proto_field_t::MISOPin, cfg.MISOPin);
+      w.add(proto_field_t::CSNPin, cfg.CSNPin);
+      w.add(proto_field_t::frequency, cfg.frequency);
+      w.add(proto_field_t::rxBandwidth, cfg.rxBandwidth);
+      w.add(proto_field_t::deviation, cfg.deviation);
+      w.add(proto_field_t::txPower, cfg.txPower);
+      w.add(proto_field_t::noiseDetection, cfg.noiseDetection);
+      w.add(proto_field_t::radioInit, cfg.radioInit);
+      break;
+    }
+  }
+}
+
+/*********************************************************************
+ * saveSettings: what the settings routes do with their request
+ ********************************************************************/
+// Applies the parts of the network section that were sent.  True when the
+// device must reboot for them, by the rules of /setNetwork.
+static bool saveNetwork(JsonObject req, uint16_t parts) {
+  bool reboot = false;
+  if(settingsSets(parts, settings_part_t::conn)) {
+    const conn_types_t connType = static_cast<conn_types_t>(req["connType"].as<uint8_t>());
+    if(connType != settings.connType) {
+      settings.connType = connType;
+      settings.save();
+      reboot = true;
+    }
+  }
+  if(settingsSets(parts, settings_part_t::wifi)) {
+    // Compared after fromJSON, which decides whether an empty passphrase keeps the stored one.
+    WifiSettings before = settings.WIFI;
+    settings.WIFI.fromJSON(req);
+    settings.WIFI.save();
+    // Not while someone is on the soft AP: the device joins the network once
+    // they have left it.
+    if(settings.connType == conn_types_t::wifi && WiFi.softAPgetStationNum() == 0 &&
+      (strcmp(before.ssid, settings.WIFI.ssid) != 0 || strcmp(before.passphrase, settings.WIFI.passphrase) != 0))
+      reboot = true;
+  }
+#if HAS_ETHERNET
+  if(settingsSets(parts, settings_part_t::ethernet)) {
+    settings.Ethernet.fromJSON(req);
+    settings.Ethernet.save();
+    if(settings.connType == conn_types_t::ethernet || settings.connType == conn_types_t::ethernetpref) reboot = true;
+  }
+#endif
+  if(settingsSets(parts, settings_part_t::ip)) {
+    settings.IP.fromJSON(req);
+    settings.IP.save();
+  }
+  return reboot;
+}
+static void runSave(const proto_reply_to_t &to, settings_save_t *save) {
+  JsonObject req = save->request.as<JsonObject>();
+  const settings_section_t section = save->section;
+  const uint16_t parts = settingsParts(section, req);
+  bool reboot = false;
+  char token[65] = "";
+  const security_types securityBefore = settings.Security.type;
+  switch(section) {
+    case settings_section_t::general:
+      if(settingsSets(parts, settings_part_t::general)) {
+        settings.fromJSON(req);
+        settings.save();
+        if(!req["hostname"].isNull()) net.updateHostname();
+      }
+      if(settingsSets(parts, settings_part_t::ntp)) {
+        settings.NTP.fromJSON(req);
+        settings.NTP.save();
+        settings.NTP.apply();
+      }
+      break;
+    case settings_section_t::network:
+      reboot = saveNetwork(req, parts);
+      break;
+    case settings_section_t::mqtt:
+      // loop() connects again with the new settings.
+      mqtt.disconnect();
+      settings.MQTT.fromJSON(req);
+      settings.MQTT.save();
+      break;
+    case settings_section_t::security:
+      settings.Security.fromJSON(req);
+      settings.Security.save();
+      if(settings.requiresAuth()) webServer.createAPIToken(IPAddress(save->ip), token);
+      break;
+    case settings_section_t::radio: {
+      // Parsed into a copy so the radio task never sees a half updated config.
+      transceiver_config_t cfg = somfy.transceiver.config;
+      cfg.fromJSON(req);
+      somfy.transceiver.setConfig(cfg);
+      break;
+    }
+  }
+  freeSettingsSave(save);
+  ProtoWriter w = protoSocket.beginReply(to);
+  w.add(proto_field_t::apiKey, token);
+  w.add(proto_field_t::rebooting, reboot);
+  protoSocket.endReply(to);
+  ESP_LOGI(TAG, "Socket [%lu] saved the %s settings%s", (unsigned long)to.client, settingsSectionName(section), reboot ? ", rebooting" : "");
+  // Under Secured every token may have changed, and a client that connected
+  // under None had none: they all connect again, the saver with the token in
+  // its reply, and the rest log in when theirs is refused.  None to None
+  // changes nobody's access.
+  const bool closeAll = section == settings_section_t::security &&
+    (securityBefore != settings.Security.type || settings.requiresAuth());
+  if(closeAll) protoSocket.closeAll(to.client);
+  else protoSocket.settingsChanged(to.client, settingsSectionName(section));
+  if(reboot) {
+    rebootDelay.rebootTime = millis() + 1000;
+    rebootDelay.reboot = true;
+  }
+}
+
+void runDeviceOp(const loop_item_t &item) {
+  switch(static_cast<proto_type_t>(item.op)) {
+    case proto_type_t::getSettings: {
+      ProtoWriter w = protoSocket.beginReply(item.replyTo);
+      writeSettings(w, static_cast<settings_section_t>(item.section));
+      protoSocket.endReply(item.replyTo);
+      break;
+    }
+    case proto_type_t::saveSettings:
+      runSave(item.replyTo, item.save);
+      break;
+    case proto_type_t::scanWifi:
+      // The networks follow as wifiNetworks once the scan is done.
+      protoSocket.wantsWifiNetworks(item.replyTo.client);
+      net.scanForClients();
+      protoSocket.beginReply(item.replyTo);
+      protoSocket.endReply(item.replyTo);
+      break;
+    case proto_type_t::beginFrequencyScan:
+      if(!somfy.transceiver.config.enabled) {
+        protoSocket.replyRefused(item.replyTo, proto_error_t::radioUnavailable, "The radio is disabled");
+        break;
+      }
+      somfy.transceiver.beginFrequencyScan();
+      protoSocket.beginReply(item.replyTo);
+      protoSocket.endReply(item.replyTo);
+      break;
+    case proto_type_t::endFrequencyScan:
+      somfy.transceiver.endFrequencyScan();
+      protoSocket.beginReply(item.replyTo);
+      protoSocket.endReply(item.replyTo);
+      break;
+    case proto_type_t::reboot:
+      ESP_LOGI(TAG, "Socket [%lu] asked for a reboot", (unsigned long)item.replyTo.client);
+      protoSocket.beginReply(item.replyTo);
+      protoSocket.endReply(item.replyTo);
+      // Time for the reply to go out.
+      rebootDelay.rebootTime = millis() + 500;
+      rebootDelay.reboot = true;
+      break;
+    default:
+      if(item.save) freeSettingsSave(item.save);
+      protoSocket.replyRefused(item.replyTo, proto_error_t::internal, "Not a device op");
+      break;
+  }
+}

src/DeviceOps.h

@@ -0,0 +1,28 @@
+#ifndef DEVICE_OPS_H
+#define DEVICE_OPS_H
+#include <stdint.h>
+#include <ArduinoJson.h>
+#include "LoopQueue.h"
+#include "ProtoSettings.h"
+
+// Socket protocol 2's settings ops and the WiFi scan, run on loop()
+// (docs/PROTOCOL.md).  loop() also reads the settings while it runs the
+// network, MQTT and the radio, so it is the one that changes them.
+
+// A saveSettings on its way to loop(), which frees it.
+struct settings_save_t {
+  settings_section_t section = settings_section_t::general;
+  // The client's address, for the token a security save answers with.
+  uint32_t ip = 0;
+  // The route's request, from expandSettingsPayload().
+  JsonDocument request;
+};
+// async_tcp: a save to fill in and hand to loop().  One runs at a time,
+// whichever client sent it: nullptr and busy while another is waiting, and
+// nullptr when there is no memory for it.
+settings_save_t *newSettingsSave(bool &busy);
+// Frees a save, run or refused, so another may follow.
+void freeSettingsSave(settings_save_t *save);
+// loop(): runs a proto_device item and replies.
+void runDeviceOp(const loop_item_t &item);
+#endif

src/ESPNetwork.cpp

@@ -24,11 +24,40 @@ extern SomfyShadeController somfy;
 extern LoopStats loopStats;
 
 static bool _apScanning = false;
+// The scan for clients: asked for, then running.  WiFi holds the results of
+// one scan at a time, so it and the roaming scan never overlap.
+static bool _clientScanWanted = false;
+static bool _clientScanning = false;
 int connectRetries = 0;
+// A client that asks during a scan gets that scan's networks.
+void ESPNetwork::scanForClients() { if(!_clientScanning) _clientScanWanted = true; }
+static void clientScanLoop(ESPNetwork &net) {
+  if(_clientScanning) {
+    const int16_t n = WiFi.scanComplete();
+    if(n == WIFI_SCAN_RUNNING) return;
+    _clientScanning = false;
+    if(n < 0) ESP_LOGW(TAG, "WiFi scan failed");
+    else ESP_LOGI(TAG, "Scanned %d networks", n);
+    protoSocket.wifiNetworks(n);
+    WiFi.scanDelete();
+    return;
+  }
+  // Not in the middle of joining a network either, as the roaming scan isn't.
+  if(!_clientScanWanted || _apScanning || net.connecting()) return;
+  _clientScanWanted = false;
+  // As /scanaps does: the soft AP's connection attempts would get in its way.
+  if(net.softAPOpened) WiFi.disconnect(false);
+  if(WiFi.scanNetworks(true, true) == WIFI_SCAN_RUNNING) _clientScanning = true;
+  else {
+    ESP_LOGW(TAG, "WiFi scan didn't start");
+    protoSocket.wifiNetworks(WIFI_SCAN_FAILED);
+  }
+}
 void ESPNetwork::end() {
   SSDP.end();
   mqtt.end();
   sockEmit.end();
+  protoSocket.end();
   delay(100);
 }
 bool ESPNetwork::setup() {
@@ -86,6 +115,8 @@ void ESPNetwork::loop() {
   //    b. WiFi: Perform synchronous scan for APs related to the SSID.  If the SSID can be found then perform
   //       the connection process for the WiFi connection.
   //    c. SoftAP: This condition retains the Soft AP because no other connection method is available.
+  // Before the connection steps, which return early while connecting.
+  clientScanLoop(*this);
   conn_types_t ctype = this->preferredConnType();
   this->connect(ctype); // Connection timeout handled in connect function as well as the opening of the Soft AP if needed.
   if(this->connecting()) return; // If we are currently attempting to connect to something then we need to bail here.
@@ -119,7 +150,7 @@ void ESPNetwork::loop() {
       }
     }
   }
-  if(!this->connecting() && !settings.WIFI.hidden) {
+  if(!this->connecting() && !settings.WIFI.hidden && !_clientScanWanted && !_clientScanning) {
     if((this->softAPOpened && WiFi.softAPgetStationNum() == 0) ||
       (!this->connected() && ctype == conn_types_t::wifi)) {
       // If the Soft AP is opened and there are no clients connected then we need to scan for an AP.  If

src/ESPNetwork.h

@@ -63,6 +63,10 @@ class ESPNetwork {
     //int getStrengthByMac(const char *mac);
     //int getStrengthBySSID(const char *ssid);
     void updateHostname();
+    // A scan of every network, hidden ones included, for the WiFi settings.
+    // It runs once no roaming scan is, and its networks go to the protocol 2
+    // clients that asked (ProtoSocket::wifiNetworks).  loop() only.
+    void scanForClients();
     bool setup();
     void loop();
     void end();

src/LoopQueue.cpp

@@ -9,6 +9,7 @@
 #include "SomfyEditor.h"
 #include "Utils.h"
 #include "outputs/ProtoSocket.h"
+#include "DeviceOps.h"
 
 static const char *TAG = "LoopQueue";
 
@@ -31,11 +32,12 @@ static restore_options_t restoreOptions(uint8_t parts) {
 void LoopQueue::begin() {
   if(!this->queue) this->queue = xQueueCreate(LOOP_QUEUE_SIZE, sizeof(loop_item_t));
 }
-bool LoopQueue::submit(const loop_item_t &item) {
+bool LoopQueue::submit(const loop_item_t &item, bool first) {
   if(!this->queue) return false;
   // async_tcp also services the sockets, so it must never wait on loop() or
   // the radio.
-  if(xQueueSend(static_cast<QueueHandle_t>(this->queue), &item, 0) != pdTRUE) {
+  QueueHandle_t q = static_cast<QueueHandle_t>(this->queue);
+  if((first ? xQueueSendToFront(q, &item, 0) : xQueueSend(q, &item, 0)) != pdTRUE) {
     ESP_LOGE(TAG, "Loop queue full, dropping item");
     return false;
   }
@@ -85,7 +87,10 @@ void LoopQueue::drain() {
         runModelEdit(item.edit);
         break;
       case loop_item_type_t::proto_read:
-        protoSocket.replyRead(item.replyTo, item.readOp, item.groupId);
+        protoSocket.replyRead(item.replyTo, item.op, item.groupId);
+        break;
+      case loop_item_type_t::proto_device:
+        runDeviceOp(item);
         break;
     }
   }

src/LoopQueue.h

@@ -26,20 +26,24 @@ enum class loop_item_type_t : uint8_t {
   load_shades,     // replace the shades with /shades.tmp
   snapshot_fill,   // fill a read's ModelSnapshot
   model_edit,      // apply an edit with ModelEditor
-  proto_read       // answer a protocol 2 read op from the shade model
+  proto_read,      // answer a protocol 2 read op from the shade model
+  proto_device     // run a protocol 2 settings op or the WiFi scan (DeviceOps)
 };
 struct model_edit_t;
+struct settings_save_t;
 struct loop_item_t {
   loop_item_type_t type = loop_item_type_t::shade_request;
   shade_request_t request;        // shade_request
   uint8_t restoreParts = 0;       // restore_shades: RESTORE_* bits
   uint8_t slot = 0;               // snapshot_fill: the SnapshotSlots slot
   model_edit_t *edit = nullptr;   // model_edit: loop() runs and frees it
-  // shade_request and proto_read from a protocol 2 client: loop() replies
-  // once it has run.
+  // shade_request, proto_read and proto_device from a protocol 2 client:
+  // loop() replies once it has run.
   proto_reply_to_t replyTo;
-  uint8_t readOp = 0;             // proto_read: the op's type
+  uint8_t op = 0;                 // proto_read and proto_device: the op's type
   uint8_t groupId = 0;            // proto_read: groupOptions' group
+  uint8_t section = 0;            // proto_device: getSettings' settings_section_t
+  settings_save_t *save = nullptr; // proto_device: saveSettings' save, which loop() frees
 };
 
 // Whether the item sends anything on air.
@@ -63,8 +67,10 @@ class LoopQueue {
   public:
     void begin();
     // Hands the item to loop() without waiting, so it is safe from any task,
-    // including async_tcp.  False when the queue is full.
-    bool submit(const loop_item_t &item);
+    // including async_tcp.  False when the queue is full.  first puts it
+    // ahead of everything waiting: an item that ends a frequency scan, which
+    // an unpair at the head waits for (holdLoopItem).
+    bool submit(const loop_item_t &item, bool first = false);
     // Runs the waiting items in order.  Only from loop().
     void drain();
   private:

src/ModelEditor.cpp

@@ -31,10 +31,9 @@ edit_changes_t editChanges(model_edit_type_t type, const edit_result_t &result)
     case model_edit_type_t::unlink_repeater:
       c.repeaters = true;
       break;
-    case model_edit_type_t::delete_room:
-    case model_edit_type_t::delete_shade:
-    case model_edit_type_t::delete_group:
-      break;
+    case model_edit_type_t::delete_room: c.removedRoomId = result.id; break;
+    case model_edit_type_t::delete_shade: c.removedShadeId = result.id; break;
+    case model_edit_type_t::delete_group: c.removedGroupId = result.id; break;
   }
   return c;
 }

src/ModelEditor.h

@@ -64,6 +64,10 @@ struct edit_changes_t {
   uint8_t roomId = 0;
   sort_list_t order = sort_list_t::none;
   bool repeaters = false;
+  // A delete: the id of what was removed.
+  uint8_t removedShadeId = 0;
+  uint8_t removedGroupId = 0;
+  uint8_t removedRoomId = 0;
 };
 edit_changes_t editChanges(model_edit_type_t type, const edit_result_t &result);
 

src/ProtoEdits.cpp

@@ -1,6 +1,7 @@
 #include <stdio.h>
 #include <string.h>
 #include "ProtoEdits.h"
+#include "ProtoParse.h"
 
 struct edit_op_t {
   proto_type_t type;
@@ -96,41 +97,9 @@ static const char *routeName(model_edit_type_t type, proto_field_t f) {
   if(type == model_edit_type_t::link_remote && f == proto_field_t::lastRollingCode) return "rollingCode";
   return protoField(f).name;
 }
-static const char *kindName(char kind) {
-  switch(kind) {
-    case 'b': return "boolean (1 or 0)";
-    case 'i': return "whole number";
-    case 'n': return "number";
-    case 's': return "string";
-    case 'l': return "list";
-    default: return "object";
-  }
-}
-
 // What the reply's desc points at; every parse runs on async_tcp, one at a
 // time.
 static char why[64];
-// One value under its field's kind: false when it isn't one.
-static bool expandValue(const proto_field_info_t &f, JsonVariantConst v, JsonVariant out) {
-  switch(f.kind) {
-    case 'b':
-      // Booleans are 1 and 0; the edit routes read true and false.
-      if(!v.is<int>() || (v.as<int>() != 0 && v.as<int>() != 1)) return false;
-      out.set(v.as<int>() == 1);
-      return true;
-    case 'i':
-      if(v.is<int64_t>() || v.is<uint64_t>()) return out.set(v);
-      return false;
-    case 'n':
-      if(v.is<float>()) return out.set(v);
-      return false;
-    case 's':
-      if(v.is<const char *>()) return out.set(v);
-      return false;
-    default:
-      return false;
-  }
-}
 const char *expandEditPayload(model_edit_type_t type, JsonObjectConst d, JsonDocument &doc) {
   doc.clear();
   switch(type) {
@@ -164,8 +133,8 @@ const char *expandEditPayload(model_edit_type_t type, JsonObjectConst d, JsonDoc
       snprintf(why, sizeof(why), "%s can't be set by this edit", f.name);
       return why;
     }
-    if(!expandValue(f, kv.value(), obj[routeName(type, field)].to<JsonVariant>())) {
-      snprintf(why, sizeof(why), "%s is not a %s", f.name, kindName(f.kind));
+    if(!protoRouteValue(f, kv.value(), obj[routeName(type, field)].to<JsonVariant>())) {
+      snprintf(why, sizeof(why), "%s is not a %s", f.name, protoKindName(f.kind));
       return why;
     }
   }

src/ProtoParse.cpp

@@ -34,3 +34,34 @@ const char *protoString(JsonObjectConst d, proto_field_t field) {
   return v.is<const char *>() ? v.as<const char *>() : nullptr;
 }
 bool protoHas(JsonObjectConst d, proto_field_t field) { return !d[protoField(field).key].isNull(); }
+
+bool protoRouteValue(const proto_field_info_t &f, JsonVariantConst v, JsonVariant out) {
+  switch(f.kind) {
+    case 'b':
+      // Booleans are 1 and 0; the routes read true and false.
+      if(!v.is<int>() || (v.as<int>() != 0 && v.as<int>() != 1)) return false;
+      out.set(v.as<int>() == 1);
+      return true;
+    case 'i':
+      if(v.is<int64_t>() || v.is<uint64_t>()) return out.set(v);
+      return false;
+    case 'n':
+      if(v.is<float>()) return out.set(v);
+      return false;
+    case 's':
+      if(v.is<const char *>()) return out.set(v);
+      return false;
+    default:
+      return false;
+  }
+}
+const char *protoKindName(char kind) {
+  switch(kind) {
+    case 'b': return "boolean (1 or 0)";
+    case 'i': return "whole number";
+    case 'n': return "number";
+    case 's': return "string";
+    case 'l': return "list";
+    default: return "object";
+  }
+}

src/ProtoParse.h

@@ -29,4 +29,10 @@ bool protoBool(JsonObjectConst d, proto_field_t field);
 // nullptr when it was left out, as a string's default is "".
 const char *protoString(JsonObjectConst d, proto_field_t field);
 bool protoHas(JsonObjectConst d, proto_field_t field);
+
+// A value sent for field f, put into out as the HTTP route reads it: booleans
+// as true and false.  False when it isn't of the field's kind.
+bool protoRouteValue(const proto_field_info_t &f, JsonVariantConst v, JsonVariant out);
+// A kind as a bad request names it: "whole number", "string" ...
+const char *protoKindName(char kind);
 #endif

src/ProtoSchema.h

@@ -58,6 +58,12 @@
   TYPE(getNextShade, 46, 'c', 'r') \
   TYPE(getNextGroup, 47, 'c', 'r') \
   TYPE(groupOptions, 48, 'c', 'r') \
+  TYPE(getSettings, 49, 'c', 'd') \
+  TYPE(saveSettings, 50, 'c', 'd') \
+  TYPE(scanWifi, 51, 'c', 'd') \
+  TYPE(beginFrequencyScan, 52, 'c', 'd') \
+  TYPE(endFrequencyScan, 53, 'c', 'd') \
+  TYPE(reboot, 54, 'c', 'd') \
   TYPE(shadeState, 128, 'd', '-') \
   TYPE(shade, 129, 'd', '-') \
   TYPE(shadeRemoved, 130, 'd', '-') \
@@ -75,7 +81,9 @@
   TYPE(ethernet, 142, 'd', '-') \
   TYPE(frame, 143, 'd', '-') \
   TYPE(frequencyScan, 144, 'd', '-') \
-  TYPE(repeaters, 145, 'd', '-')
+  TYPE(repeaters, 145, 'd', '-') \
+  TYPE(settingsChanged, 146, 'd', '-') \
+  TYPE(wifiNetworks, 147, 'd', '-')
 
 // Fields, in one dictionary for every type.  FIELD(name, key, kind, default):
 // kind is b bool (sent as 1/0), i integer, n number with a fraction, s string,
@@ -187,7 +195,64 @@
   FIELD(scanning, "sc", 'b', 0) \
   FIELD(testFreq, "tf", 'n', 0) \
   FIELD(testRssi, "tr", 'i', 0) \
-  FIELD(frequency, "fq", 'n', 0)
+  FIELD(frequency, "fq", 'n', 0) \
+  /* the settings ops: which section, and what a save did */ \
+  FIELD(section, "se", 's', 0) \
+  FIELD(apiKey, "ak", 's', 0) \
+  FIELD(rebooting, "rb", 'b', 0) \
+  /* general */ \
+  FIELD(ssdpBroadcast, "sb", 'b', 0) \
+  FIELD(ntpServer, "ns", 's', 0) \
+  FIELD(posixZone, "pz", 's', 0) \
+  /* network: the connection the settings ask for (connType is the one in \
+     use), WiFi, Ethernet and the IP address */ \
+  FIELD(connMode, "cn", 'i', 0) \
+  FIELD(hasEthernet, "he", 'b', 0) \
+  FIELD(passphrase, "pp", 's', 0) \
+  FIELD(hasPassphrase, "hq", 'b', 0) \
+  FIELD(roaming, "ro", 'b', 0) \
+  FIELD(hidden, "hd", 'b', 0) \
+  FIELD(boardType, "bo", 'i', 0) \
+  FIELD(phyType, "ph", 'i', 0) \
+  FIELD(CLKMode, "ck", 'i', 0) \
+  FIELD(phyAddress, "pa", 'i', 0) \
+  FIELD(PWRPin, "pw", 'i', 0) \
+  FIELD(MDCPin, "mc", 'i', 0) \
+  FIELD(MDIOPin, "mi", 'i', 0) \
+  FIELD(dhcp, "dh", 'b', 0) \
+  FIELD(ip, "ip", 's', 0) \
+  FIELD(gateway, "gw", 's', 0) \
+  FIELD(subnet, "su", 's', 0) \
+  FIELD(dns1, "d1", 's', 0) \
+  FIELD(dns2, "d2", 's', 0) \
+  /* MQTT and security; hostname is the broker's in MQTT */ \
+  FIELD(enabled, "en", 'b', 0) \
+  FIELD(pubDisco, "pd", 'b', 0) \
+  FIELD(protocol, "pt", 's', 0) \
+  FIELD(port, "po", 'i', 0) \
+  FIELD(username, "un", 's', 0) \
+  FIELD(password, "ps", 's', 0) \
+  FIELD(hasPassword, "hw", 'b', 0) \
+  FIELD(rootTopic, "rt", 's', 0) \
+  FIELD(discoTopic, "dt", 's', 0) \
+  FIELD(securityType, "sy", 'i', 0) \
+  FIELD(pin, "pn", 's', 0) \
+  FIELD(hasPin, "hi", 'b', 0) \
+  /* the radio: bitLength, proto, frequency and enabled as above */ \
+  FIELD(TXPin, "tx", 'i', 0) \
+  FIELD(RXPin, "rx", 'i', 0) \
+  FIELD(SCKPin, "sk", 'i', 0) \
+  FIELD(MOSIPin, "mo", 'i', 0) \
+  FIELD(MISOPin, "ms", 'i', 0) \
+  FIELD(CSNPin, "cs", 'i', 0) \
+  FIELD(rxBandwidth, "bw", 'n', 0) \
+  FIELD(deviation, "dv", 'n', 0) \
+  FIELD(txPower, "tp", 'i', 0) \
+  FIELD(noiseDetection, "nd", 'b', 0) \
+  FIELD(radioInit, "ri", 'b', 0) \
+  /* wifiNetworks: each network has ssid, channel and strength as above */ \
+  FIELD(networks, "nw", 'l', 0) \
+  FIELD(macAddress, "ma", 's', 0)
 
 // Error codes in a reply.  ERR(name, number)
 #define PROTO_ERRORS(ERR) \
@@ -231,7 +296,8 @@ enum class proto_op_t : char {
   shadeRequest = 'q',  // a command: a shade request loop() runs
   remoteFrame = 'f',   // sendRemoteCommand: a frame that goes to the radio at once
   edit = 'e',          // an edit loop() applies
-  read = 'r'           // a read loop() answers from the shade model
+  read = 'r',          // a read loop() answers from the shade model
+  device = 'd'         // a settings, scan or reboot op loop() runs
 };
 const proto_field_info_t &protoField(proto_field_t field);
 const proto_field_info_t *protoFieldByKey(const char *key);

src/ProtoSettings.cpp

@@ -0,0 +1,205 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "ProtoSettings.h"
+#include "ProtoParse.h"
+
+static const char *const SECTIONS[] = {"general", "network", "mqtt", "security", "radio"};
+const char *settingsSectionName(settings_section_t section) { return SECTIONS[static_cast<uint8_t>(section)]; }
+
+static settings_section_t sectionOf(settings_part_t part) {
+  switch(part) {
+    case settings_part_t::general:
+    case settings_part_t::ntp: return settings_section_t::general;
+    case settings_part_t::conn:
+    case settings_part_t::wifi:
+    case settings_part_t::ethernet:
+    case settings_part_t::ip: return settings_section_t::network;
+    case settings_part_t::mqtt: return settings_section_t::mqtt;
+    case settings_part_t::security: return settings_section_t::security;
+    default: return settings_section_t::radio;
+  }
+}
+
+// What a save may set.  lo..hi is a whole number's range, a number's in
+// hundredths, and hi a string's longest length: what the setting holds, so the
+// route's fromJSON neither cuts it short nor wraps it.  The has* fields and
+// radioInit are only ever sent.
+struct settings_field_t {
+  settings_part_t part;
+  proto_field_t field;
+  const char *route;  // the name the route reads; nullptr when it is the field's
+  int32_t lo;
+  int32_t hi;
+};
+static const settings_field_t FIELDS[] = {
+  {settings_part_t::general, proto_field_t::hostname, nullptr, 0, 31},
+  {settings_part_t::general, proto_field_t::ssdpBroadcast, nullptr, 0, 0},
+  {settings_part_t::ntp, proto_field_t::ntpServer, nullptr, 0, 64},
+  {settings_part_t::ntp, proto_field_t::posixZone, nullptr, 0, 63},
+  // wifi, ethernet and ethernet with WiFi as the fallback (conn_types_t).
+  {settings_part_t::conn, proto_field_t::connMode, "connType", 1, 3},
+  {settings_part_t::wifi, proto_field_t::ssid, nullptr, 0, 64},
+  {settings_part_t::wifi, proto_field_t::passphrase, nullptr, 0, 64},
+  {settings_part_t::wifi, proto_field_t::roaming, nullptr, 0, 0},
+  {settings_part_t::wifi, proto_field_t::hidden, nullptr, 0, 0},
+  {settings_part_t::ethernet, proto_field_t::boardType, nullptr, 0, 255},
+  {settings_part_t::ethernet, proto_field_t::phyType, nullptr, 0, 255},
+  {settings_part_t::ethernet, proto_field_t::CLKMode, nullptr, 0, 3},
+  {settings_part_t::ethernet, proto_field_t::phyAddress, nullptr, -1, 31},
+  {settings_part_t::ethernet, proto_field_t::PWRPin, nullptr, -1, 48},
+  {settings_part_t::ethernet, proto_field_t::MDCPin, nullptr, 0, 48},
+  {settings_part_t::ethernet, proto_field_t::MDIOPin, nullptr, 0, 48},
+  {settings_part_t::ip, proto_field_t::dhcp, nullptr, 0, 0},
+  {settings_part_t::ip, proto_field_t::ip, nullptr, 0, 15},
+  {settings_part_t::ip, proto_field_t::gateway, nullptr, 0, 15},
+  {settings_part_t::ip, proto_field_t::subnet, nullptr, 0, 15},
+  {settings_part_t::ip, proto_field_t::dns1, nullptr, 0, 15},
+  {settings_part_t::ip, proto_field_t::dns2, nullptr, 0, 15},
+  {settings_part_t::mqtt, proto_field_t::enabled, nullptr, 0, 0},
+  {settings_part_t::mqtt, proto_field_t::pubDisco, nullptr, 0, 0},
+  {settings_part_t::mqtt, proto_field_t::protocol, nullptr, 0, 9},
+  {settings_part_t::mqtt, proto_field_t::hostname, nullptr, 0, 64},
+  {settings_part_t::mqtt, proto_field_t::port, nullptr, 1, 65535},
+  {settings_part_t::mqtt, proto_field_t::username, nullptr, 0, 32},
+  {settings_part_t::mqtt, proto_field_t::password, nullptr, 0, 32},
+  {settings_part_t::mqtt, proto_field_t::rootTopic, nullptr, 0, 64},
+  {settings_part_t::mqtt, proto_field_t::discoTopic, nullptr, 0, 64},
+  // None, PIN or password (security_types).
+  {settings_part_t::security, proto_field_t::securityType, "type", 0, 2},
+  {settings_part_t::security, proto_field_t::username, nullptr, 0, 32},
+  {settings_part_t::security, proto_field_t::password, nullptr, 0, 32},
+  {settings_part_t::security, proto_field_t::pin, nullptr, 0, 4},
+  // 56 or 80, checked on its own.
+  {settings_part_t::radio, proto_field_t::bitLength, "type", 56, 80},
+  // RTS, RTW, RTV, GP Relay or GP Remote (radio_proto), checked on its own.
+  {settings_part_t::radio, proto_field_t::proto, nullptr, 0, 9},
+  {settings_part_t::radio, proto_field_t::enabled, nullptr, 0, 0},
+  {settings_part_t::radio, proto_field_t::TXPin, nullptr, 0, 48},
+  {settings_part_t::radio, proto_field_t::RXPin, nullptr, 0, 48},
+  {settings_part_t::radio, proto_field_t::SCKPin, nullptr, 0, 48},
+  {settings_part_t::radio, proto_field_t::MOSIPin, nullptr, 0, 48},
+  {settings_part_t::radio, proto_field_t::MISOPin, nullptr, 0, 48},
+  {settings_part_t::radio, proto_field_t::CSNPin, nullptr, 0, 48},
+  // What the CC1101 can be set to, in MHz and kHz.
+  {settings_part_t::radio, proto_field_t::frequency, nullptr, 30000, 92800},
+  {settings_part_t::radio, proto_field_t::rxBandwidth, nullptr, 5803, 81250},
+  {settings_part_t::radio, proto_field_t::deviation, nullptr, 158, 38085},
+  {settings_part_t::radio, proto_field_t::txPower, nullptr, -30, 12},
+  {settings_part_t::radio, proto_field_t::noiseDetection, nullptr, 0, 0}
+};
+static const char *routeName(const settings_field_t &s) { return s.route ? s.route : protoField(s.field).name; }
+static const settings_field_t *saved(settings_section_t section, proto_field_t field) {
+  for(const settings_field_t &s : FIELDS)
+    if(s.field == field && sectionOf(s.part) == section) return &s;
+  return nullptr;
+}
+
+const char *parseSettingsSection(JsonObjectConst d, settings_section_t &section) {
+  const char *name = protoString(d, proto_field_t::section);
+  if(!name) return "section is missing";
+  for(uint8_t i = 0; i < sizeof(SECTIONS) / sizeof(SECTIONS[0]); i++) {
+    if(strcmp(SECTIONS[i], name) != 0) continue;
+    section = static_cast<settings_section_t>(i);
+    return nullptr;
+  }
+  return "Unknown section";
+}
+
+// Four numbers 0-255 between dots.
+static bool isAddress(const char *s) {
+  for(int part = 0; part < 4; part++) {
+    if(part > 0 && *s++ != '.') return false;
+    if(*s < '0' || *s > '9') return false;
+    char *end;
+    const long n = strtol(s, &end, 10);
+    if(n > 255 || end - s > 3) return false;
+    s = end;
+  }
+  return *s == '\0';
+}
+
+// What the reply's desc points at; every parse runs on async_tcp, one at a
+// time.
+static char why[64];
+// A value already of its field's kind: nullptr when it is in range.
+static const char *checkValue(const settings_field_t &s, const proto_field_info_t &f, JsonVariant value) {
+  if(f.kind == 'i') {
+    const bool inRange = value.is<int64_t>() && value.as<int64_t>() >= s.lo && value.as<int64_t>() <= s.hi;
+    if(s.field == proto_field_t::bitLength) {
+      if(inRange && (value.as<int>() == 56 || value.as<int>() == 80)) return nullptr;
+      snprintf(why, sizeof(why), "%s must be 56 or 80", f.name);
+      return why;
+    }
+    if(s.field == proto_field_t::proto) {
+      if(inRange && (value.as<int>() <= 2 || value.as<int>() >= 8)) return nullptr;
+      snprintf(why, sizeof(why), "%s must be 0, 1, 2, 8 or 9", f.name);
+      return why;
+    }
+    if(inRange) return nullptr;
+    snprintf(why, sizeof(why), "%s must be %ld-%ld", f.name, (long)s.lo, (long)s.hi);
+    return why;
+  }
+  if(f.kind == 'n') {
+    // Half a hundredth either way, as 58.03 is a little under it in a float.
+    const double hundredths = value.as<double>() * 100;
+    if(hundredths > s.lo - 0.5 && hundredths < s.hi + 0.5) return nullptr;
+    snprintf(why, sizeof(why), "%s must be %ld.%02ld-%ld.%02ld", f.name, (long)(s.lo / 100), (long)(s.lo % 100),
+      (long)(s.hi / 100), (long)(s.hi % 100));
+    return why;
+  }
+  if(f.kind != 's') return nullptr;
+  const char *text = value.as<const char *>();
+  if(strlen(text) > static_cast<size_t>(s.hi)) {
+    snprintf(why, sizeof(why), "%s is longer than %ld characters", f.name, (long)s.hi);
+    return why;
+  }
+  if(s.part != settings_part_t::ip) return nullptr;
+  // The settings send 0.0.0.0 as "", so "" sets it back.
+  if(!text[0]) value.set("0.0.0.0");
+  else if(!isAddress(text)) {
+    snprintf(why, sizeof(why), "%s is not an IP address", f.name);
+    return why;
+  }
+  return nullptr;
+}
+
+const char *expandSettingsPayload(JsonObjectConst d, bool hasEthernet, settings_section_t &section, JsonDocument &doc) {
+  doc.clear();
+  const char *bad = parseSettingsSection(d, section);
+  if(bad) return bad;
+  JsonObject obj = doc.to<JsonObject>();
+  for(JsonPairConst kv : d) {
+    proto_field_t field;
+    if(!protoFieldOfKey(kv.key().c_str(), field)) {
+      snprintf(why, sizeof(why), "Unknown field %s", kv.key().c_str());
+      return why;
+    }
+    if(field == proto_field_t::section) continue;
+    const proto_field_info_t &f = protoField(field);
+    const settings_field_t *s = saved(section, field);
+    if(!s) {
+      snprintf(why, sizeof(why), "%s can't be saved in %s", f.name, settingsSectionName(section));
+      return why;
+    }
+    if(s->part == settings_part_t::ethernet && !hasEthernet) return "This device has no Ethernet";
+    JsonVariant value = obj[routeName(*s)].to<JsonVariant>();
+    if(!protoRouteValue(f, kv.value(), value)) {
+      snprintf(why, sizeof(why), "%s is not a %s", f.name, protoKindName(f.kind));
+      return why;
+    }
+    bad = checkValue(*s, f, value);
+    if(bad) return bad;
+  }
+  if(obj.size() == 0) return "Nothing to save";
+  return doc.overflowed() ? "Too large" : nullptr;
+}
+
+uint16_t settingsParts(settings_section_t section, JsonObjectConst request) {
+  uint16_t parts = 0;
+  for(JsonPairConst kv : request)
+    for(const settings_field_t &s : FIELDS)
+      if(sectionOf(s.part) == section && strcmp(routeName(s), kv.key().c_str()) == 0)
+        parts |= 1u << static_cast<uint8_t>(s.part);
+  return parts;
+}

src/ProtoSettings.h

@@ -0,0 +1,50 @@
+#ifndef PROTO_SETTINGS_H
+#define PROTO_SETTINGS_H
+#include <stdint.h>
+#include <ArduinoJson.h>
+#include "ProtoSchema.h"
+
+// Socket protocol 2's settings ops (docs/PROTOCOL.md): the sections that
+// getSettings and saveSettings name, and what a save's payload becomes.  A
+// save is the settings route's request: its payload, in full names, is what
+// loop() hands to the settings' own fromJSON(), so the rules stay in one
+// place.  Kept free of the settings and the web server so it can be unit
+// tested on the PC (pio test -e native).
+
+enum class settings_section_t : uint8_t {
+  general,   // hostname, SSDP and the clock
+  network,   // the connection, WiFi, Ethernet and the IP address
+  mqtt,
+  security,
+  radio
+};
+// The settings each part of a section belongs to, which loop() saves only
+// when a field of it was sent.
+enum class settings_part_t : uint8_t {
+  general,   // ConfigSettings: hostname, ssdpBroadcast
+  ntp,       // NTPSettings
+  conn,      // ConfigSettings: connType
+  wifi,      // WifiSettings
+  ethernet,  // EthernetSettings
+  ip,        // IPSettings
+  mqtt,      // MQTTSettings
+  security,  // SecuritySettings
+  radio      // transceiver_config_t
+};
+const char *settingsSectionName(settings_section_t section);
+// The section that payload d names in section.  nullptr when it is good;
+// otherwise what is wrong with it, for a bad request reply.
+const char *parseSettingsSection(JsonObjectConst d, settings_section_t &section);
+// saveSettings: the section d names, and in doc its fields under the names
+// the route reads.  Each is checked against its kind and its range or length,
+// so nothing is cut short or wraps.  An address may be "" to clear it.  As
+// with every op, only what was sent is in it; a field the section doesn't
+// save, such as hasPassword, is refused.  Without hasEthernet the Ethernet
+// fields are refused too.  nullptr when it is good; otherwise what is wrong.
+const char *expandSettingsPayload(JsonObjectConst d, bool hasEthernet, settings_section_t &section, JsonDocument &doc);
+// The parts that a save's request (from expandSettingsPayload) sets, as bits
+// 1 << settings_part_t.  A route name such as hostname means another setting
+// in another section, hence the section.
+uint16_t settingsParts(settings_section_t section, JsonObjectConst request);
+inline bool settingsSets(uint16_t parts, settings_part_t part) { return parts & (1u << static_cast<uint8_t>(part)); }
+#endif

src/ShadeController.cpp

@@ -6,7 +6,6 @@
 #include "ConfigFile.h"
 #include "ModelEditor.h"
 #include "ShadeChanges.h"
-#include "outputs/ProtoSocket.h"
 
 static const char *TAG = "Somfy";
 
@@ -291,7 +290,6 @@ bool SomfyShadeController::deleteShade(uint8_t shadeId) {
   for(uint8_t i = 0; i < SOMFY_MAX_SHADES; i++) {
     if(this->shades[i].getShadeId() == shadeId) {
       shades[i].emitState("shadeRemoved");
-      protoSocket.shadeRemoved(shadeId);
       shades[i].unpublish();
       this->shades[i].clear();
     }
@@ -322,7 +320,6 @@ bool SomfyShadeController::deleteRoom(uint8_t roomId) {
         }
       }
       rooms[i].emitState("roomRemoved");
-      protoSocket.roomRemoved(roomId);
       this->rooms[i].clear();
     }
   }
@@ -338,7 +335,6 @@ bool SomfyShadeController::deleteGroup(uint8_t groupId) {
         if(shade) shade->configChanged();
       }
       groups[i].emitState("groupRemoved");
-      protoSocket.groupRemoved(groupId);
       groups[i].unpublish();
       this->groups[i].clear();
     }

src/SomfyJson.cpp

@@ -262,13 +262,18 @@ static void markEdited(const model_edit_t &edit, const edit_result_t &result) {
   if(room) protoSocket.room(*room);
   if(c.order != sort_list_t::none) protoSocket.order(c.order);
   if(c.repeaters) protoSocket.repeaters();
+  if(c.removedShadeId) protoSocket.shadeRemoved(c.removedShadeId);
+  if(c.removedGroupId) protoSocket.groupRemoved(c.removedGroupId);
+  if(c.removedRoomId) protoSocket.roomRemoved(c.removedRoomId);
 }
 void runModelEdit(model_edit_t *edit) {
   const edit_result_t result = applyModelEdit(*edit);
   if(!result.ok) ESP_LOGW(TAG, "Edit %u refused: %s", static_cast<uint8_t>(edit->type), result.desc);
-  else markEdited(*edit, result);
+  // The reply first: the edit's events follow it (docs/PROTOCOL.md), so a
+  // client that waits for the reply doesn't read past them.
   if(edit->replyTo.client) protoSocket.replyEdit(edit->replyTo, edit->type, result);
   else emitRequestResult(edit->requestId, result);
+  if(result.ok) markEdited(*edit, result);
   cancelModelEdit(edit);
 }
 

src/outputs/ProtoSocket.cpp

@@ -12,6 +12,8 @@
 #include "ProtoOps.h"
 #include "ProtoEdits.h"
 #include "ProtoAssembly.h"
+#include "ProtoSettings.h"
+#include "DeviceOps.h"
 #include "SomfyEditor.h"
 #include "LoopQueue.h"
 #include "SomfySnapshot.h"
@@ -213,12 +215,49 @@ static void queueEditOp(AsyncWebSocketClient *client, const proto_message_t &msg
 static void queueReadOp(AsyncWebSocketClient *client, const proto_message_t &msg, JsonObjectConst d) {
   loop_item_t item;
   item.type = loop_item_type_t::proto_read;
-  item.readOp = msg.type;
+  item.op = msg.type;
   const char *bad = parseReadPayload(msg.type, d, item.groupId);
   if(bad) { replyError(client, msg.id, proto_error_t::badRequest, bad); return; }
   item.replyTo = {client->id(), msg.id};
   if(!loopQueue.submit(item)) replyError(client, msg.id, proto_error_t::busy, "Command queue full");
 }
+// A settings op or the WiFi scan: loop() runs it and replies.
+static void queueDeviceOp(AsyncWebSocketClient *client, const proto_message_t &msg, JsonObjectConst d) {
+  loop_item_t item;
+  item.type = loop_item_type_t::proto_device;
+  item.op = msg.type;
+  item.replyTo = {client->id(), msg.id};
+  if(msg.type == static_cast<uint8_t>(proto_type_t::getSettings)) {
+    settings_section_t section;
+    const char *bad = parseSettingsSection(d, section);
+    if(bad) { replyError(client, msg.id, proto_error_t::badRequest, bad); return; }
+    item.section = static_cast<uint8_t>(section);
+  }
+  else if(msg.type == static_cast<uint8_t>(proto_type_t::saveSettings)) {
+    bool busy;
+    settings_save_t *save = newSettingsSave(busy);
+    if(!save) {
+      if(busy) replyError(client, msg.id, proto_error_t::busy, "Another save is running, send it again");
+      else replyError(client, msg.id, proto_error_t::internal, "Out of memory");
+      return;
+    }
+    const char *bad = expandSettingsPayload(d, HAS_ETHERNET, save->section, save->request);
+    if(bad) {
+      freeSettingsSave(save);
+      replyError(client, msg.id, proto_error_t::badRequest, bad);
+      return;
+    }
+    // A security save answers with the client's new token.
+    save->ip = client->remoteIP();
+    item.save = save;
+  }
+  // An unpair waits at the head of the queue while a frequency scan runs, so
+  // the scan's end goes ahead of it.
+  const bool first = msg.type == static_cast<uint8_t>(proto_type_t::endFrequencyScan);
+  if(loopQueue.submit(item, first)) return;
+  if(item.save) freeSettingsSave(item.save);
+  replyError(client, msg.id, proto_error_t::busy, "Command queue full");
+}
 static bool authorized(AsyncWebServerRequest *request) {
   if(settings.Security.type == security_types::None) return true;
   const AsyncWebParameter *key = request->getParam("apikey");
@@ -304,8 +343,10 @@ static void onMessage(AsyncWebSocketClient *client, const char *text, size_t len
     case proto_op_t::read:
       queueReadOp(client, msg, doc["d"]);
       return;
+    case proto_op_t::device:
+      queueDeviceOp(client, msg, doc["d"]);
+      return;
     case proto_op_t::none:
-      // The settings ops arrive with the protocol core's next PR.
       replyError(client, msg.id, proto_error_t::unknownOp, "Unknown message type");
       return;
   }
@@ -374,6 +415,7 @@ void ProtoSocket::begin(AsyncWebServer &server) {
   ws.onEvent(ProtoSocket::wsEvent);
   server.addHandler(&ws);
 }
+void ProtoSocket::end() { ws.closeAll(1001, "Restarting"); }
 bool ProtoSocket::hasClients() const {
   for(const proto_slot_t &s : slots)
     if(s.clientId.load()) return true;
@@ -384,6 +426,7 @@ bool ProtoSocket::sending() const { return sockEmit.onLoopTask() && this->hasCli
 /*********************************************************************
  * loop(): syncs
  ********************************************************************/
+static void closeOnceSent();
 static void sendSyncFrame(proto_slot_t &s) {
   LiveSource src;
   char *buf = socketFrameBuffer();
@@ -424,6 +467,7 @@ void ProtoSocket::loop() {
     writeErrorReply(json, 0, proto_error_t::badRequest, "The message took too long to arrive");
     ws.text(stalled, buf, json.length());
   }
+  closeOnceSent();
   ws.cleanupClients(PROTO_MAX_CLIENTS);
 }
 
@@ -526,6 +570,27 @@ void ProtoSocket::replyRead(const proto_reply_to_t &to, uint8_t op, uint8_t grou
   sendReply(to, json, buf, sizeof(buf));
 }
 
+ProtoWriter ProtoSocket::beginReply(const proto_reply_to_t &to) {
+  eventJson.setBuffer(socketFrameBuffer(), PROTO_MAX_FRAME);
+  beginFrame(eventJson, proto_type_t::reply, to.id);
+  ProtoWriter w(eventJson);
+  w.add(proto_field_t::ok, true);
+  return w;
+}
+void ProtoSocket::endReply(const proto_reply_to_t &to) {
+  if(!sockEmit.onLoopTask() || to.id == 0) return;
+  endFrame(eventJson);
+  sendReply(to, eventJson, socketFrameBuffer(), PROTO_MAX_FRAME);
+}
+void ProtoSocket::replyRefused(const proto_reply_to_t &to, proto_error_t code, const char *desc) {
+  if(!sockEmit.onLoopTask() || to.id == 0) return;
+  char buf[192];
+  BufferedJsonFormatter json;
+  json.setBuffer(buf, sizeof(buf));
+  writeErrorReply(json, to.id, code, desc);
+  sendReply(to, json, buf, sizeof(buf));
+}
+
 /*********************************************************************
  * loop(): events
  ********************************************************************/
@@ -657,6 +722,84 @@ void ProtoSocket::ethernet() {
   writeEthernet(w);
   sendEvent();
 }
+void ProtoSocket::settingsChanged(uint32_t savedBy, const char *section) {
+  if(!this->sending()) return;
+  ProtoWriter(beginEvent(proto_type_t::settingsChanged)).add(proto_field_t::section, section);
+  if(!finishEvent()) return;
+  for(proto_slot_t &s : slots) {
+    const uint32_t id = s.clientId.load();
+    if(id && id != savedBy) ws.text(id, socketFrameBuffer(), eventJson.length());
+  }
+}
+// The client that saved the security settings, closed once its reply has
+// gone: a close frame goes ahead of the messages waiting to be sent.
+// loop() only.
+static uint32_t closeWhenSent = 0;
+static uint32_t closeWhenSentSince = 0;
+#define CLOSE_WHEN_SENT_MAX_MS 2000
+void ProtoSocket::closeAll(uint32_t savedBy) {
+  if(!sockEmit.onLoopTask()) return;
+  for(proto_slot_t &s : slots) {
+    const uint32_t id = s.clientId.load();
+    if(id && id != savedBy) ws.close(id);
+  }
+  closeWhenSent = savedBy;
+  closeWhenSentSince = millis();
+}
+static void closeOnceSent() {
+  if(!closeWhenSent) return;
+  AsyncWebSocketClient *client = ws.client(closeWhenSent);
+  if(client && client->queueLen() > 0 && millis() - closeWhenSentSince < CLOSE_WHEN_SENT_MAX_MS) return;
+  if(client) client->close();
+  closeWhenSent = 0;
+}
+// The clients waiting for a WiFi scan.  loop() only.
+static uint32_t scanAskers[PROTO_MAX_CLIENTS];
+// The most networks wifiNetworks lists, so the event fits a frame.
+#define PROTO_MAX_NETWORKS 30
+void ProtoSocket::wantsWifiNetworks(uint32_t clientId) {
+  if(!sockEmit.onLoopTask()) return;
+  uint32_t *spare = nullptr;
+  for(uint32_t &id : scanAskers) {
+    if(id == clientId) return;
+    // A client that has gone is skipped when the event is sent, so its place
+    // can go to another.
+    if(!spare && (id == 0 || !slotOf(slots, PROTO_MAX_CLIENTS, id))) spare = &id;
+  }
+  if(spare) *spare = clientId;
+}
+void ProtoSocket::wifiNetworks(int16_t count) {
+  if(!sockEmit.onLoopTask()) return;
+  bool asked = false;
+  for(uint32_t id : scanAskers) asked |= id != 0;
+  if(!asked) return;
+  // Each asker was told the networks would follow, so when they don't fit
+  // fewer go: names that need escaping can make one five times longer.
+  bool fits = false;
+  for(uint8_t most = PROTO_MAX_NETWORKS; !fits && most > 0; most /= 2) {
+    ProtoWriter w(beginEvent(proto_type_t::wifiNetworks));
+    w.beginList(proto_field_t::networks);
+    uint8_t listed = 0;
+    for(int16_t i = 0; i < count && listed < most; i++) {
+      // As /scanaps lists them: named networks that can be heard.
+      if(WiFi.SSID(i).length() == 0 || WiFi.RSSI(i) < -95) continue;
+      w.beginItem();
+      w.add(proto_field_t::ssid, WiFi.SSID(i).c_str());
+      w.add(proto_field_t::channel, (int32_t)WiFi.channel(i));
+      w.add(proto_field_t::strength, (int32_t)WiFi.RSSI(i));
+      w.add(proto_field_t::macAddress, WiFi.BSSIDstr(i).c_str());
+      w.endItem();
+      listed++;
+    }
+    w.endList();
+    endFrame(eventJson);
+    fits = !eventJson.overflowed();
+  }
+  for(uint32_t &id : scanAskers) {
+    if(fits && id) ws.text(id, socketFrameBuffer(), eventJson.length());
+    id = 0;
+  }
+}
 bool ProtoSocket::subscribed(proto_feed_t feed) const {
   // The feed is formatted into the event buffer, which is loop()'s.
   if(!sockEmit.onLoopTask()) return false;

src/outputs/ProtoSocket.h

@@ -24,6 +24,9 @@ class ProtoSocket {
     void begin(AsyncWebServer &server);
     // Moves each client's sync on.  loop() only.
     void loop();
+    // Closes every client, before a reboot: a restart drops the connections
+    // without a word, so clients wouldn't know they had gone.
+    void end();
     bool hasClients() const;
     // The reply to a command op once loop() has run its shade request.
     void replyRequest(const proto_reply_to_t &to, const shade_request_t &req, shade_request_result_t result);
@@ -31,6 +34,22 @@ class ProtoSocket {
     void replyEdit(const proto_reply_to_t &to, model_edit_type_t type, const edit_result_t &result);
     // The reply to a read op, from the shade model.  loop() only.
     void replyRead(const proto_reply_to_t &to, uint8_t op, uint8_t groupId);
+    // Any other reply formatted on loop(): ok, and the fields written between
+    // the two.  One too large for a frame goes as an internal error.
+    ProtoWriter beginReply(const proto_reply_to_t &to);
+    void endReply(const proto_reply_to_t &to);
+    void replyRefused(const proto_reply_to_t &to, proto_error_t code, const char *desc);
+    // settingsChanged, to every client but the one that saved them.
+    void settingsChanged(uint32_t savedBy, const char *section);
+    // Closes every client, as a security save does: their tokens may no
+    // longer be good, or they had none.  savedBy is closed once its reply has
+    // gone, so it reads its new token first.
+    void closeAll(uint32_t savedBy);
+    // wifiNetworks goes to the clients that asked for a scan since the last
+    // one, once it is done: count is WiFi.scanComplete(), and the networks
+    // are WiFi's scan results.
+    void wantsWifiNetworks(uint32_t clientId);
+    void wifiNetworks(int16_t count);
     // Events, loop() only.
     void shadeState(SomfyShade &shade);
     void shade(SomfyShade &shade);

test/e2e/README.md

@@ -12,8 +12,11 @@ Runs against a real device on your network, by hand, never in CI. It checks sock
 | `--password`, `--username` / `--pin` | the credentials of a Secured device |
 | `--radio --shade <id>` | also send real commands: that shade moves a little and back |
 | `--edit` | also add a room and delete it again |
+| `--reboot` | also reboot the device and wait for it to come back |
 | `--record` | save hello, the sync and the schema to `test/fixtures/proto2/`, for the web UI and Home Assistant tests |
 
-A plain run sends nothing on air, changes no settings and doesn't reboot the device. The command tests send ops that are refused before they reach the radio, and one `setPositions` that sets a shade to the position the sync gave: that drops the fraction of a percent the shade's believed position may have had. The edit tests send edits that are refused, and one shade sort order set to the order it has, which saves the shades again unchanged. It does fill every socket slot for a few seconds, and replays a page load.
+A plain run sends nothing on air, changes no setting (it saves only values the device already has) and doesn't reboot the device. The command tests send ops that are refused before they reach the radio, and one `setPositions` that sets a shade to the position the sync gave: that drops the fraction of a percent the shade's believed position may have had. The edit tests send edits that are refused, and one shade sort order set to the order it has, which saves the shades again unchanged. The settings tests send saves that are refused, and one that saves the NTP server and time zone the device already has. The frequency scan test pauses the radio's sending for a second or two, as a scan does. It does fill every socket slot for a few seconds, and replays a page load.
+
+The web UI holds one of the device's 5 socket slots for each open tab. The tests that fill every slot skip while one is taken, so close the UI's tabs before a run.
 
 The tests run in whatever security mode the device is in; the suite never changes it. To test both modes, run the suite once in each.

test/e2e/conftest.py

@@ -18,6 +18,7 @@ def pytest_addoption(parser):
     parser.addoption("--radio", action="store_true", help="also run the tests that send commands on air")
     parser.addoption("--shade", type=int, default=None, help="with --radio: the id of the shade the commands may move")
     parser.addoption("--edit", action="store_true", help="also run the edits that change the shade model, then undo them")
+    parser.addoption("--reboot", action="store_true", help="also run the test that reboots the device")
 
 
 @pytest.fixture(scope="session")

test/e2e/test_settings.py

@@ -0,0 +1,154 @@
+"""The settings ops, the WiFi and frequency scans, and reboot (docs/PROTOCOL.md).
+Without --reboot nothing changes: saves are refused, or save what is already
+there, and nothing goes on air."""
+import time
+
+import pytest
+
+from proto_client import ProtoClient, run_sync
+from test_commands import is_ok, op, synced_model
+from test_messages import error_name
+
+SETTINGS_OPS = ("getSettings", "saveSettings", "scanWifi", "beginFrequencyScan", "endFrequencyScan", "reboot")
+SECTIONS = ("general", "network", "mqtt", "security", "radio")
+SECRETS = ("password", "passphrase", "pin")
+
+
+def desc(schema, reply):
+    return reply["d"].get(schema.keys["desc"])
+
+
+def settings(c, schema, section):
+    reply = op(c, schema, "getSettings", {"section": section})
+    assert is_ok(schema, reply), desc(schema, reply)
+    return schema.expand(reply["d"])
+
+
+def test_every_settings_op_is_in_the_schema(schema):
+    for name in SETTINGS_OPS:
+        assert 16 <= schema.type_numbers[name] <= 127, name
+    for name in ("settingsChanged", "wifiNetworks"):
+        assert schema.type_numbers[name] >= 128, name
+
+
+@pytest.mark.parametrize("section", SECTIONS)
+def test_every_section_reads_without_its_secrets(connect, schema, section):
+    c, _, _ = synced_model(connect, schema)
+    got = settings(c, schema, section)
+    for name in SECRETS:
+        assert name not in got, name
+
+
+def test_the_sections_hold_what_the_routes_send(connect, schema):
+    c, _, _ = synced_model(connect, schema)
+    assert settings(c, schema, "general").get("hostname")
+    assert settings(c, schema, "radio").get("bitLength", schema.default("bitLength")) in (56, 80)
+    assert settings(c, schema, "mqtt").get("port", 0) > 0
+
+
+def test_an_unknown_section_is_a_bad_request(connect, schema):
+    c, _, _ = synced_model(connect, schema)
+    reply = op(c, schema, "getSettings", {"section": "wifi"})
+    assert error_name(schema, reply) == "badRequest"
+    assert desc(schema, reply) == "Unknown section"
+
+
+@pytest.mark.parametrize("section, fields, why", [
+    ("mqtt", {}, "Nothing to save"),
+    ("mqtt", {"hasPassword": 1}, "hasPassword can't be saved in mqtt"),
+    ("general", {"ssid": "x"}, "ssid can't be saved in general"),
+    ("network", {"connMode": 9}, "connMode must be 1-3"),
+    ("network", {"ip": "10.0.0"}, "ip is not an IP address"),
+    ("radio", {"bitLength": 60}, "bitLength must be 56 or 80"),
+    ("security", {"pin": "12345"}, "pin is longer than 4 characters"),
+])
+def test_a_save_is_checked_before_it_runs(connect, schema, section, fields, why):
+    c, _, _ = synced_model(connect, schema)
+    reply = op(c, schema, "saveSettings", dict(fields, section=section))
+    assert error_name(schema, reply) == "badRequest"
+    assert desc(schema, reply) == why
+
+
+def test_a_save_tells_the_other_clients(connect, schema):
+    c, _, _ = synced_model(connect, schema)
+    other, _, _ = synced_model(connect, schema)
+    general = settings(c, schema, "general")
+    # What is there already, so nothing changes.
+    same = {name: general.get(name, schema.default(name)) for name in ("ntpServer", "posixZone")}
+    reply = op(c, schema, "saveSettings", dict(same, section="general"))
+    assert is_ok(schema, reply), desc(schema, reply)
+    assert not schema.expand(reply["d"]).get("rebooting")
+    changed = other.recv_until(lambda f: f["t"] == schema.type_numbers["settingsChanged"])[-1]
+    assert schema.expand(changed["d"])["section"] == "general"
+    assert settings(c, schema, "general") == general
+
+
+def test_a_wifi_scan_lists_the_networks(connect, schema):
+    c = connect(schema=schema.hash)
+    _, frames, _ = run_sync(c)
+    header = next(schema.expand(f["d"][schema.keys["header"]]) for f in frames if schema.keys["header"] in f["d"])
+    # The connection in use: Ethernet with WiFi as the fallback may be on either.
+    if header.get("connType") != "Wifi":
+        pytest.skip("the device isn't on WiFi")
+    assert is_ok(schema, op(c, schema, "scanWifi", {}))
+    found = c.recv_until(lambda f: f["t"] == schema.type_numbers["wifiNetworks"], timeout=20)[-1]
+    networks = schema.expand(found["d"]).get("networks", [])
+    assert networks, "no networks, not even the one the device is on"
+    for n in networks:
+        assert n["ssid"] and n["macAddress"]
+
+
+def test_a_frequency_scan_starts_and_ends(connect, schema):
+    c, _, _ = synced_model(connect, schema)
+    scan = schema.type_numbers["frequencyScan"]
+    assert is_ok(schema, op(c, schema, "subscribe", {"feed": schema.feed_numbers["frequencyScan"]}))
+    reply = op(c, schema, "beginFrequencyScan", {})
+    if not is_ok(schema, reply):
+        assert error_name(schema, reply) == "radioUnavailable"
+        pytest.skip("the radio is disabled")
+    try:
+        started = c.recv_until(lambda f: f["t"] == scan, timeout=5)[-1]
+        assert schema.expand(started["d"]).get("scanning")
+    finally:
+        assert is_ok(schema, op(c, schema, "endFrequencyScan", {}))
+    ended = c.recv_until(lambda f: f["t"] == scan and not schema.expand(f["d"]).get("scanning"), timeout=5)[-1]
+    assert ended
+
+
+def test_reboot(request, host, token, connect, schema):
+    if not request.config.getoption("--reboot"):
+        pytest.skip("reboots the device: pass --reboot")
+    c = connect(schema=schema.hash)
+    _, frames, _ = run_sync(c)
+    uptime_before = uptime(schema, frames)
+    assert is_ok(schema, op(c, schema, "reboot", {}))
+    assert c.closed.wait(15), "the socket stayed open"
+    # The device closes its sockets just before it restarts, and may still
+    # take a connection in between: that one never gets its hello.
+    time.sleep(3)
+    end = time.time() + 90
+    while True:
+        assert time.time() < end, "the device didn't come back"
+        try:
+            back = ProtoClient(host, params={"apikey": token} if token else None, timeout=3)
+        except OSError:
+            time.sleep(2)
+            continue
+        try:
+            if back.status == 101:
+                hello, frames, _ = run_sync(back, timeout=10)
+                assert hello["d"]["sh"] == schema.hash
+                assert uptime(schema, frames) < uptime_before, "the device didn't restart"
+                return
+        except TimeoutError:
+            pass  # not back yet
+        finally:
+            back.close()
+        time.sleep(2)
+
+
+def uptime(schema, frames):
+    """The uptime, in ms, from the status a sync ends with."""
+    key = schema.keys["status"]
+    status = next(f["d"][key] for f in frames if key in f["d"])
+    return schema.expand(status)["uptime"]

test/e2e/test_survival.py

@@ -2,6 +2,7 @@
 (docs/adr/0005-one-websocket-protocol.md): every client reconnecting at once,
 and a browser loading the page."""
 import threading
+import time
 import urllib.request
 
 import pytest
@@ -29,7 +30,24 @@ def watch_heap(host):
     assert after["min"] >= HEAP_FLOOR, "the heap fell to %u bytes (floor %u)" % (after["min"], HEAP_FLOOR)
 
 
-def test_every_client_reconnecting_at_once(connect, schema, watch_heap):
+@pytest.fixture
+def all_slots_free(host, token):
+    """Skips unless every socket slot is free.  The web UI holds one per open
+    tab, which would leave these tests a client short."""
+    clients = [ProtoClient(host, params={"apikey": token} if token else None) for _ in range(MAX_CLIENTS)]
+    try:
+        time.sleep(1)
+        taken = sum(1 for c in clients if c.closed.is_set())
+    finally:
+        for c in clients:
+            c.close()
+    # The device frees a slot when it sees the disconnect.
+    time.sleep(0.5)
+    if taken:
+        pytest.skip("%u socket slot(s) in use, likely by the web UI: close its tabs" % taken)
+
+
+def test_every_client_reconnecting_at_once(connect, schema, watch_heap, all_slots_free):
     clients = [connect() for _ in range(MAX_CLIENTS)]
     errors = []
 
@@ -44,7 +62,7 @@ def test_every_client_reconnecting_at_once(connect, schema, watch_heap):
     assert not errors, errors
 
 
-def test_one_client_too_many_is_closed(connect, schema):
+def test_one_client_too_many_is_closed(connect, schema, all_slots_free):
     clients = [connect(schema=schema.hash) for _ in range(MAX_CLIENTS)]
     for c in clients:
         run_sync(c)
@@ -54,7 +72,7 @@ def test_one_client_too_many_is_closed(connect, schema):
 
 def test_page_load(host, token, connect, watch_heap):
     """The static files in parallel, as a browser fetches them, then the socket."""
-    files = ["/", "/main.css", "/widgets.css", "/icons.css", "/index.js", "/icon.png"]
+    files = ["/", "/app.css", "/index.js", "/icon.png"]
     errors = []
 
     def get(path):

test/fixtures/proto2/hello.json

@@ -3,6 +3,6 @@
  "t": 1,
  "d": {
   "fw": "v3.0.14",
-  "sh": "0fa739cc"
+  "sh": "16fdffc7"
  }
 }

test/fixtures/proto2/schema.json

@@ -1,5 +1,5 @@
 {
- "hash": "0fa739cc",
+ "hash": "16fdffc7",
  "types": {
   "1": "hello",
   "2": "reply",
@@ -18,6 +18,35 @@
   "23": "unpairShade",
   "24": "setRollingCode",
   "25": "sendRemoteCommand",
+  "26": "addRoom",
+  "27": "addShade",
+  "28": "addGroup",
+  "29": "saveRoom",
+  "30": "saveShade",
+  "31": "saveGroup",
+  "32": "deleteRoom",
+  "33": "deleteShade",
+  "34": "deleteGroup",
+  "35": "linkRemote",
+  "36": "unlinkRemote",
+  "37": "linkToGroup",
+  "38": "unlinkFromGroup",
+  "39": "linkRepeater",
+  "40": "unlinkRepeater",
+  "41": "setPaired",
+  "42": "roomSortOrder",
+  "43": "shadeSortOrder",
+  "44": "groupSortOrder",
+  "45": "getNextRoom",
+  "46": "getNextShade",
+  "47": "getNextGroup",
+  "48": "groupOptions",
+  "49": "getSettings",
+  "50": "saveSettings",
+  "51": "scanWifi",
+  "52": "beginFrequencyScan",
+  "53": "endFrequencyScan",
+  "54": "reboot",
   "128": "shadeState",
   "129": "shade",
   "130": "shadeRemoved",
@@ -35,7 +64,9 @@
   "142": "ethernet",
   "143": "frame",
   "144": "frequencyScan",
-  "145": "repeaters"
+  "145": "repeaters",
+  "146": "settingsChanged",
+  "147": "wifiNetworks"
  },
  "fields": {
   "D": {
@@ -108,16 +139,31 @@
    "kind": "i",
    "default": 0
   },
+  "ak": {
+   "name": "apiKey",
+   "kind": "s",
+   "default": 0
+  },
   "bl": {
    "name": "bitLength",
    "kind": "i",
    "default": 56
   },
+  "bo": {
+   "name": "boardType",
+   "kind": "i",
+   "default": 0
+  },
   "bt": {
    "name": "bits",
    "kind": "i",
    "default": 0
   },
+  "bw": {
+   "name": "rxBandwidth",
+   "kind": "n",
+   "default": 0
+  },
   "c": {
    "name": "lastRollingCode",
    "kind": "i",
@@ -128,11 +174,21 @@
    "kind": "i",
    "default": 0
   },
+  "ck": {
+   "name": "CLKMode",
+   "kind": "i",
+   "default": 0
+  },
   "cm": {
    "name": "command",
    "kind": "s",
    "default": 0
   },
+  "cn": {
+   "name": "connMode",
+   "kind": "i",
+   "default": 0
+  },
   "co": {
    "name": "connected",
    "kind": "b",
@@ -143,6 +199,11 @@
    "kind": "s",
    "default": 0
   },
+  "cs": {
+   "name": "CSNPin",
+   "kind": "i",
+   "default": 0
+  },
   "ct": {
    "name": "connType",
    "kind": "s",
@@ -153,6 +214,31 @@
    "kind": "i",
    "default": 0
   },
+  "d1": {
+   "name": "dns1",
+   "kind": "s",
+   "default": 0
+  },
+  "d2": {
+   "name": "dns2",
+   "kind": "s",
+   "default": 0
+  },
+  "dh": {
+   "name": "dhcp",
+   "kind": "b",
+   "default": 0
+  },
+  "dt": {
+   "name": "discoTopic",
+   "kind": "s",
+   "default": 0
+  },
+  "dv": {
+   "name": "deviation",
+   "kind": "n",
+   "default": 0
+  },
   "e": {
    "name": "code",
    "kind": "i",
@@ -163,6 +249,11 @@
    "kind": "i",
    "default": 0
   },
+  "en": {
+   "name": "enabled",
+   "kind": "b",
+   "default": 0
+  },
   "f": {
    "name": "flags",
    "kind": "i",
@@ -228,21 +319,51 @@
    "kind": "i",
    "default": 0
   },
+  "gw": {
+   "name": "gateway",
+   "kind": "s",
+   "default": 0
+  },
   "h": {
    "name": "header",
    "kind": "o",
    "default": 0
   },
+  "hd": {
+   "name": "hidden",
+   "kind": "b",
+   "default": 0
+  },
+  "he": {
+   "name": "hasEthernet",
+   "kind": "b",
+   "default": 0
+  },
+  "hi": {
+   "name": "hasPin",
+   "kind": "b",
+   "default": 0
+  },
   "hn": {
    "name": "hostname",
    "kind": "s",
    "default": 0
   },
+  "hq": {
+   "name": "hasPassphrase",
+   "kind": "b",
+   "default": 0
+  },
   "hs": {
    "name": "hwsync",
    "kind": "i",
    "default": 0
   },
+  "hw": {
+   "name": "hasPassword",
+   "kind": "b",
+   "default": 0
+  },
   "i": {
    "name": "paired",
    "kind": "b",
@@ -253,6 +374,11 @@
    "kind": "b",
    "default": 0
   },
+  "ip": {
+   "name": "ip",
+   "kind": "s",
+   "default": 0
+  },
   "is": {
    "name": "ids",
    "kind": "l",
@@ -293,16 +419,41 @@
    "kind": "i",
    "default": -1
   },
+  "ma": {
+   "name": "macAddress",
+   "kind": "s",
+   "default": 0
+  },
+  "mc": {
+   "name": "MDCPin",
+   "kind": "i",
+   "default": 0
+  },
   "md": {
    "name": "model",
    "kind": "s",
    "default": 0
   },
+  "mi": {
+   "name": "MDIOPin",
+   "kind": "i",
+   "default": 0
+  },
   "mn": {
    "name": "minHeap",
    "kind": "i",
    "default": 0
   },
+  "mo": {
+   "name": "MOSIPin",
+   "kind": "i",
+   "default": 0
+  },
+  "ms": {
+   "name": "MISOPin",
+   "kind": "i",
+   "default": 0
+  },
   "mx": {
    "name": "maxHeap",
    "kind": "i",
@@ -313,6 +464,21 @@
    "kind": "s",
    "default": 0
   },
+  "nd": {
+   "name": "noiseDetection",
+   "kind": "b",
+   "default": 0
+  },
+  "ns": {
+   "name": "ntpServer",
+   "kind": "s",
+   "default": 0
+  },
+  "nw": {
+   "name": "networks",
+   "kind": "l",
+   "default": 0
+  },
   "o": {
    "name": "sortOrder",
    "kind": "i",
@@ -323,16 +489,66 @@
    "kind": "i",
    "default": 0
   },
+  "pa": {
+   "name": "phyAddress",
+   "kind": "i",
+   "default": 0
+  },
+  "pd": {
+   "name": "pubDisco",
+   "kind": "b",
+   "default": 0
+  },
+  "ph": {
+   "name": "phyType",
+   "kind": "i",
+   "default": 0
+  },
+  "pn": {
+   "name": "pin",
+   "kind": "s",
+   "default": 0
+  },
+  "po": {
+   "name": "port",
+   "kind": "i",
+   "default": 0
+  },
+  "pp": {
+   "name": "passphrase",
+   "kind": "s",
+   "default": 0
+  },
   "pr": {
    "name": "proto",
    "kind": "i",
    "default": 0
   },
+  "ps": {
+   "name": "password",
+   "kind": "s",
+   "default": 0
+  },
+  "pt": {
+   "name": "protocol",
+   "kind": "s",
+   "default": 0
+  },
   "pu": {
    "name": "pulses",
    "kind": "l",
    "default": 0
   },
+  "pw": {
+   "name": "PWRPin",
+   "kind": "i",
+   "default": 0
+  },
+  "pz": {
+   "name": "posixZone",
+   "kind": "s",
+   "default": 0
+  },
   "q": {
    "name": "tiltPosition",
    "kind": "i",
@@ -343,11 +559,26 @@
    "kind": "i",
    "default": 0
   },
+  "rb": {
+   "name": "rebooting",
+   "kind": "b",
+   "default": 0
+  },
   "re": {
    "name": "repeat",
    "kind": "i",
    "default": 0
   },
+  "ri": {
+   "name": "radioInit",
+   "kind": "b",
+   "default": 0
+  },
+  "ro": {
+   "name": "roaming",
+   "kind": "b",
+   "default": 0
+  },
   "rp": {
    "name": "repeats",
    "kind": "i",
@@ -358,11 +589,21 @@
    "kind": "i",
    "default": 0
   },
+  "rt": {
+   "name": "rootTopic",
+   "kind": "s",
+   "default": 0
+  },
   "rw": {
    "name": "rawCmd",
    "kind": "i",
    "default": 0
   },
+  "rx": {
+   "name": "RXPin",
+   "kind": "i",
+   "default": 0
+  },
   "s": {
    "name": "shadeId",
    "kind": "i",
@@ -373,6 +614,11 @@
    "kind": "i",
    "default": 0
   },
+  "sb": {
+   "name": "ssdpBroadcast",
+   "kind": "b",
+   "default": 0
+  },
   "sc": {
    "name": "scanning",
    "kind": "b",
@@ -383,6 +629,11 @@
    "kind": "s",
    "default": 0
   },
+  "se": {
+   "name": "section",
+   "kind": "s",
+   "default": 0
+  },
   "sg": {
    "name": "strength",
    "kind": "i",
@@ -393,6 +644,11 @@
    "kind": "s",
    "default": 0
   },
+  "sk": {
+   "name": "SCKPin",
+   "kind": "i",
+   "default": 0
+  },
   "sm": {
    "name": "simMy",
    "kind": "b",
@@ -428,11 +684,21 @@
    "kind": "i",
    "default": 0
   },
+  "su": {
+   "name": "subnet",
+   "kind": "s",
+   "default": 0
+  },
   "sv": {
    "name": "serverId",
    "kind": "s",
    "default": 0
   },
+  "sy": {
+   "name": "securityType",
+   "kind": "i",
+   "default": 0
+  },
   "t": {
    "name": "target",
    "kind": "i",
@@ -448,16 +714,31 @@
    "kind": "i",
    "default": 0
   },
+  "tp": {
+   "name": "txPower",
+   "kind": "i",
+   "default": 0
+  },
   "tr": {
    "name": "testRssi",
    "kind": "i",
    "default": 0
   },
+  "tx": {
+   "name": "TXPin",
+   "kind": "i",
+   "default": 0
+  },
   "u": {
    "name": "tiltTarget",
    "kind": "i",
    "default": 0
   },
+  "un": {
+   "name": "username",
+   "kind": "s",
+   "default": 0
+  },
   "up": {
    "name": "uptime",
    "kind": "i",
@@ -468,6 +749,11 @@
    "kind": "b",
    "default": 0
   },
+  "vc": {
+   "name": "validation",
+   "kind": "i",
+   "default": 0
+  },
   "w": {
    "name": "tiltDirection",
    "kind": "i",

test/fixtures/proto2/sync.json

@@ -72,6 +72,122 @@
      25,
      "sendRemoteCommand"
     ],
+    [
+     26,
+     "addRoom"
+    ],
+    [
+     27,
+     "addShade"
+    ],
+    [
+     28,
+     "addGroup"
+    ],
+    [
+     29,
+     "saveRoom"
+    ],
+    [
+     30,
+     "saveShade"
+    ],
+    [
+     31,
+     "saveGroup"
+    ],
+    [
+     32,
+     "deleteRoom"
+    ],
+    [
+     33,
+     "deleteShade"
+    ],
+    [
+     34,
+     "deleteGroup"
+    ],
+    [
+     35,
+     "linkRemote"
+    ],
+    [
+     36,
+     "unlinkRemote"
+    ],
+    [
+     37,
+     "linkToGroup"
+    ],
+    [
+     38,
+     "unlinkFromGroup"
+    ],
+    [
+     39,
+     "linkRepeater"
+    ],
+    [
+     40,
+     "unlinkRepeater"
+    ],
+    [
+     41,
+     "setPaired"
+    ],
+    [
+     42,
+     "roomSortOrder"
+    ],
+    [
+     43,
+     "shadeSortOrder"
+    ],
+    [
+     44,
+     "groupSortOrder"
+    ],
+    [
+     45,
+     "getNextRoom"
+    ],
+    [
+     46,
+     "getNextShade"
+    ],
+    [
+     47,
+     "getNextGroup"
+    ],
+    [
+     48,
+     "groupOptions"
+    ],
+    [
+     49,
+     "getSettings"
+    ],
+    [
+     50,
+     "saveSettings"
+    ],
+    [
+     51,
+     "scanWifi"
+    ],
+    [
+     52,
+     "beginFrequencyScan"
+    ],
+    [
+     53,
+     "endFrequencyScan"
+    ],
+    [
+     54,
+     "reboot"
+    ],
     [
      128,
      "shadeState"
@@ -143,6 +259,14 @@
     [
      145,
      "repeaters"
+    ],
+    [
+     146,
+     "settingsChanged"
+    ],
+    [
+     147,
+     "wifiNetworks"
     ]
    ],
    "F": [
@@ -512,6 +636,12 @@
      "b",
      0
     ],
+    [
+     "vc",
+     "validation",
+     "i",
+     0
+    ],
     [
      "xr",
      "maxRooms",
@@ -721,6 +851,314 @@
      "frequency",
      "n",
      0
+    ],
+    [
+     "se",
+     "section",
+     "s",
+     0
+    ],
+    [
+     "ak",
+     "apiKey",
+     "s",
+     0
+    ],
+    [
+     "rb",
+     "rebooting",
+     "b",
+     0
+    ],
+    [
+     "sb",
+     "ssdpBroadcast",
+     "b",
+     0
+    ],
+    [
+     "ns",
+     "ntpServer",
+     "s",
+     0
+    ],
+    [
+     "pz",
+     "posixZone",
+     "s",
+     0
+    ],
+    [
+     "cn",
+     "connMode",
+     "i",
+     0
+    ],
+    [
+     "he",
+     "hasEthernet",
+     "b",
+     0
+    ],
+    [
+     "pp",
+     "passphrase",
+     "s",
+     0
+    ],
+    [
+     "hq",
+     "hasPassphrase",
+     "b",
+     0
+    ],
+    [
+     "ro",
+     "roaming",
+     "b",
+     0
+    ],
+    [
+     "hd",
+     "hidden",
+     "b",
+     0
+    ],
+    [
+     "bo",
+     "boardType",
+     "i",
+     0
+    ],
+    [
+     "ph",
+     "phyType",
+     "i",
+     0
+    ],
+    [
+     "ck",
+     "CLKMode",
+     "i",
+     0
+    ],
+    [
+     "pa",
+     "phyAddress",
+     "i",
+     0
+    ],
+    [
+     "pw",
+     "PWRPin",
+     "i",
+     0
+    ],
+    [
+     "mc",
+     "MDCPin",
+     "i",
+     0
+    ],
+    [
+     "mi",
+     "MDIOPin",
+     "i",
+     0
+    ],
+    [
+     "dh",
+     "dhcp",
+     "b",
+     0
+    ],
+    [
+     "ip",
+     "ip",
+     "s",
+     0
+    ],
+    [
+     "gw",
+     "gateway",
+     "s",
+     0
+    ],
+    [
+     "su",
+     "subnet",
+     "s",
+     0
+    ],
+    [
+     "d1",
+     "dns1",
+     "s",
+     0
+    ]
+   ]
+  }
+ },
+ {
+  "v": 2,
+  "t": 4,
+  "d": {
+   "F": [
+    [
+     "d2",
+     "dns2",
+     "s",
+     0
+    ],
+    [
+     "en",
+     "enabled",
+     "b",
+     0
+    ],
+    [
+     "pd",
+     "pubDisco",
+     "b",
+     0
+    ],
+    [
+     "pt",
+     "protocol",
+     "s",
+     0
+    ],
+    [
+     "po",
+     "port",
+     "i",
+     0
+    ],
+    [
+     "un",
+     "username",
+     "s",
+     0
+    ],
+    [
+     "ps",
+     "password",
+     "s",
+     0
+    ],
+    [
+     "hw",
+     "hasPassword",
+     "b",
+     0
+    ],
+    [
+     "rt",
+     "rootTopic",
+     "s",
+     0
+    ],
+    [
+     "dt",
+     "discoTopic",
+     "s",
+     0
+    ],
+    [
+     "sy",
+     "securityType",
+     "i",
+     0
+    ],
+    [
+     "pn",
+     "pin",
+     "s",
+     0
+    ],
+    [
+     "hi",
+     "hasPin",
+     "b",
+     0
+    ],
+    [
+     "tx",
+     "TXPin",
+     "i",
+     0
+    ],
+    [
+     "rx",
+     "RXPin",
+     "i",
+     0
+    ],
+    [
+     "sk",
+     "SCKPin",
+     "i",
+     0
+    ],
+    [
+     "mo",
+     "MOSIPin",
+     "i",
+     0
+    ],
+    [
+     "ms",
+     "MISOPin",
+     "i",
+     0
+    ],
+    [
+     "cs",
+     "CSNPin",
+     "i",
+     0
+    ],
+    [
+     "bw",
+     "rxBandwidth",
+     "n",
+     0
+    ],
+    [
+     "dv",
+     "deviation",
+     "n",
+     0
+    ],
+    [
+     "tp",
+     "txPower",
+     "i",
+     0
+    ],
+    [
+     "nd",
+     "noiseDetection",
+     "b",
+     0
+    ],
+    [
+     "ri",
+     "radioInit",
+     "b",
+     0
+    ],
+    [
+     "nw",
+     "networks",
+     "l",
+     0
+    ],
+    [
+     "ma",
+     "macAddress",
+     "s",
+     0
     ]
    ],
    "E": [
@@ -775,15 +1213,30 @@
     "sv": "serverId 2",
     "hn": "hostname 3",
     "md": "ESPSomfyRTS",
+    "cp": "s3",
     "fw": "v3.0.14",
     "ct": "Wifi"
    },
+   "R": [
+    {
+     "r": 1,
+     "n": "name 4",
+     "o": 1
+    },
+    {
+     "r": 2,
+     "n": "name 5",
+     "o": 2
+    }
+   ],
    "S": [
     {
      "s": 1,
-     "c": 1048580,
-     "n": "name 5",
-     "a": 1048582,
+     "p": 7,
+     "t": 7,
+     "c": 1048582,
+     "n": "name 7",
+     "a": 1048584,
      "U": 18000,
      "D": 19000,
      "Y": 7000,
@@ -795,20 +1248,19 @@
      "o": 1,
      "lk": [
       {
-       "a": 1048583,
-       "c": 1048584
+       "a": 1048585
       },
       {
-       "a": 1048585,
-       "c": 1048586
+       "a": 1048586,
+       "c": 1048587
       }
      ]
     },
     {
      "s": 2,
-     "c": 1048587,
-     "n": "name 12",
-     "a": 1048589,
+     "c": 1048588,
+     "n": "name 13",
+     "a": 1048590,
      "U": 18000,
      "D": 19000,
      "Y": 7000,
@@ -820,17 +1272,18 @@
      "o": 2,
      "lk": [
       {
-       "a": 1048590,
-       "c": 1048591
+       "a": 1048591,
+       "c": 1048592
       },
       {
-       "a": 1048592,
-       "c": 1048593
+       "a": 1048593
       }
      ]
     },
     {
      "s": 3,
+     "p": 100,
+     "t": 100,
      "c": 1048594,
      "n": "name 19",
      "a": 1048596,
@@ -845,20 +1298,19 @@
      "o": 3,
      "lk": [
       {
-       "a": 1048597,
-       "c": 1048598
+       "a": 1048597
       },
       {
-       "a": 1048599,
-       "c": 1048600
+       "a": 1048598,
+       "c": 1048599
       }
      ]
     },
     {
      "s": 4,
-     "c": 1048601,
-     "n": "name 26",
-     "a": 1048603,
+     "c": 1048600,
+     "n": "name 25",
+     "a": 1048602,
      "U": 18000,
      "D": 19000,
      "Y": 7000,
@@ -870,20 +1322,19 @@
      "o": 4,
      "lk": [
       {
-       "a": 1048604,
-       "c": 1048605
+       "a": 1048603
       },
       {
-       "a": 1048606,
-       "c": 1048607
+       "a": 1048604,
+       "c": 1048605
       }
      ]
     },
     {
      "s": 5,
-     "c": 1048608,
-     "n": "name 33",
-     "a": 1048610,
+     "c": 1048606,
+     "n": "name 31",
+     "a": 1048608,
      "U": 18000,
      "D": 19000,
      "Y": 7000,
@@ -895,30 +1346,19 @@
      "o": 5,
      "lk": [
       {
-       "a": 1048611,
-       "c": 1048612
+       "a": 1048609,
+       "c": 1048610
       },
       {
-       "a": 1048613,
-       "c": 1048614
+       "a": 1048611
       }
      ]
-    }
-   ]
-  }
     },
- {
-  "v": 2,
-  "t": 4,
-  "d": {
-   "S": [
     {
      "s": 6,
-     "p": 100,
-     "t": 100,
-     "c": 1048615,
-     "n": "name 40",
-     "a": 1048617,
+     "c": 1048612,
+     "n": "name 37",
+     "a": 1048614,
      "U": 25000,
      "D": 25000,
      "Y": 7000,
@@ -930,20 +1370,19 @@
      "o": 6,
      "lk": [
       {
-       "a": 1048618,
-       "c": 1048619
+       "a": 1048615,
+       "c": 1048616
       },
       {
-       "a": 1048620,
-       "c": 1048621
+       "a": 1048617
       }
      ]
     },
     {
      "s": 7,
-     "c": 1048622,
-     "n": "name 47",
-     "a": 1048624,
+     "c": 1048618,
+     "n": "name 43",
+     "a": 1048620,
      "U": 25000,
      "D": 25000,
      "Y": 7000,
@@ -955,20 +1394,19 @@
      "o": 7,
      "lk": [
       {
-       "a": 1048625,
-       "c": 1048626
+       "a": 1048621,
+       "c": 1048622
       },
       {
-       "a": 1048627,
-       "c": 1048628
+       "a": 1048623
       }
      ]
     },
     {
      "s": 8,
-     "c": 1048629,
-     "n": "name 54",
-     "a": 1048631,
+     "c": 1048624,
+     "n": "name 49",
+     "a": 1048626,
      "U": 18000,
      "D": 19000,
      "Y": 7000,
@@ -980,20 +1418,19 @@
      "o": 8,
      "lk": [
       {
-       "a": 1048632,
-       "c": 1048633
+       "a": 1048627,
+       "c": 1048628
       },
       {
-       "a": 1048634,
-       "c": 1048635
+       "a": 1048629
       }
      ]
     },
     {
      "s": 9,
-     "c": 1048636,
-     "n": "name 61",
-     "a": 1048638,
+     "c": 1048630,
+     "n": "name 55",
+     "a": 1048632,
      "U": 18000,
      "D": 19000,
      "Y": 7000,
@@ -1005,20 +1442,19 @@
      "o": 9,
      "lk": [
       {
-       "a": 1048639,
-       "c": 1048640
+       "a": 1048633,
+       "c": 1048634
       },
       {
-       "a": 1048641,
-       "c": 1048642
+       "a": 1048635
       }
      ]
     },
     {
      "s": 10,
-     "c": 1048643,
-     "n": "name 68",
-     "a": 1048645,
+     "c": 1048636,
+     "n": "name 61",
+     "a": 1048638,
      "U": 18000,
      "D": 19000,
      "Y": 7000,
@@ -1030,12 +1466,11 @@
      "o": 10,
      "lk": [
       {
-       "a": 1048646,
-       "c": 1048647
+       "a": 1048639,
+       "c": 1048640
       },
       {
-       "a": 1048648,
-       "c": 1048649
+       "a": 1048641
       }
      ]
     }
@@ -1043,9 +1478,9 @@
    "G": [
     {
      "g": 1,
-     "n": "name 74",
-     "a": 1048651,
-     "c": 1048652,
+     "n": "name 66",
+     "a": 1048643,
+     "c": 1048644,
      "bl": 80,
      "pr": 1,
      "fc": 1,
@@ -1065,9 +1500,9 @@
     },
     {
      "g": 2,
-     "n": "name 77",
-     "a": 1048654,
-     "c": 1048655,
+     "n": "name 69",
+     "a": 1048646,
+     "c": 1048647,
      "bl": 80,
      "pr": 1,
      "fc": 1,
@@ -1080,13 +1515,15 @@
     }
    ],
    "Z": {
-    "fr": 106640,
-    "mx": 49140,
-    "mn": 61988,
-    "to": 266820,
-    "up": 29406,
-    "sd": "ssid 80",
-    "sg": -69,
+    "fr": 148760,
+    "mx": 77812,
+    "mn": 102424,
+    "to": 294616,
+    "up": 30144,
+    "lp": 2862,
+    "lx": 106737,
+    "sd": "ssid 72",
+    "sg": -52,
     "ch": 6
    }
   }

test/js/client.test.js

@@ -0,0 +1,339 @@
+// The web UI's ProtoClient (data-src/js/client.js): the recorded hello and
+// sync replayed through a stand-in socket.
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const { ProtoClient, ProtoError } = require('../../data-src/js/client.js');
+const { hello, sync, syncWithoutSchema, recordedItems, recordedSchema, FakeSocket, FakeTimers, FakeStorage } = require('./helpers.js');
+
+function client(opts) {
+    FakeSocket.reset();
+    const timers = new FakeTimers();
+    const c = new ProtoClient(Object.assign({ host: 'device', WebSocket: FakeSocket, setTimeout: timers.setTimeout,
+        clearTimeout: timers.clearTimeout, storage: null }, opts || {}));
+    c.timers = timers;
+    return c;
+}
+// Connects and plays the recorded hello and sync.
+function connected(opts) {
+    const c = client(opts);
+    c.connect();
+    const ws = FakeSocket.last;
+    ws.open();
+    ws.deliver(hello);
+    for (const f of sync) ws.deliver(f);
+    return { c, ws };
+}
+const schema = recordedSchema();
+const T = schema.typeNumbers;
+function reply(id, d) { return { v: 2, t: T.reply, id: id, d: d }; }
+
+test('the client connects with its token and syncs, sending next after each sync frame', () => {
+    const c = client({ token: 'abc 123' });
+    c.connect();
+    const ws = FakeSocket.last;
+    assert.equal(ws.url, 'ws://device/ws?apikey=abc%20123');
+    ws.open();
+    ws.deliver(hello);
+    for (const f of sync) ws.deliver(f);
+    const syncFrames = sync.filter((f) => f.t === T.sync).length;
+    assert.equal(ws.sent.filter((f) => f.t === T.next).length, syncFrames);
+    assert.ok(c.ready);
+    assert.equal(c.fw, hello.d.fw);
+    assert.equal(c.model.shades.size, recordedItems('S').length);
+});
+
+test('synced comes once the sync has ended', () => {
+    const c = client();
+    let synced = 0;
+    c.on('synced', () => synced++);
+    c.connect();
+    const ws = FakeSocket.last;
+    ws.open();
+    ws.deliver(hello);
+    for (const f of sync.filter((f) => f.t === T.sync)) ws.deliver(f);
+    assert.equal(synced, 0);
+    ws.deliver(sync.find((f) => f.t === T.syncEnd));
+    assert.equal(synced, 1);
+});
+
+test('an op resolves with its reply', async () => {
+    const { c, ws } = connected();
+    const p = c.op('getSettings', { section: 'radio' });
+    const sent = ws.sent[ws.sent.length - 1];
+    assert.deepEqual(sent, { v: 2, t: T.getSettings, id: sent.id, d: { se: 'radio' } });
+    ws.deliver(reply(sent.id, { k: 1, bl: 80, en: 1 }));
+    assert.deepEqual(await p, { ok: true, bitLength: 80, enabled: true });
+});
+
+test('a refused op rejects with the error by its name', async () => {
+    const { c, ws } = connected();
+    const p = c.op('saveSettings', { section: 'mqtt' });
+    const id = ws.sent[ws.sent.length - 1].id;
+    ws.deliver(reply(id, { e: schema.errorNumbers.badRequest, x: 'Nothing to save' }));
+    await assert.rejects(p, (err) => err instanceof ProtoError && err.code === 'badRequest' && err.desc === 'Nothing to save');
+});
+
+test('an op is refused before the sync, and when its reply never comes', async () => {
+    const c = client();
+    await assert.rejects(c.op('reboot'), (err) => err.code === 'notConnected');
+    const { c: c2 } = connected();
+    const p = c2.op('reboot');
+    c2.timers.runAll();
+    await assert.rejects(p, (err) => err.code === 'timeout');
+});
+
+test('an op of a name or field the device lacks is refused here', async () => {
+    const { c } = connected();
+    await assert.rejects(c.op('noSuchOp'), (err) => err.code === 'unknownOp');
+    await assert.rejects(c.op('reboot', { noSuchField: 1 }), (err) => err.code === 'badRequest');
+});
+
+test('ops waiting when the socket closes are refused, and the client connects again', async () => {
+    const { c, ws } = connected();
+    const p = c.op('scanWifi');
+    let closed = 0;
+    c.on('closed', () => closed++);
+    ws.drop();
+    await assert.rejects(p, (err) => err.code === 'closed');
+    assert.equal(closed, 1);
+    assert.equal(c.ready, false);
+    assert.deepEqual(c.timers.runAll(), [500]);
+    assert.notEqual(FakeSocket.last, ws);
+});
+
+test('a socket refused before it opened says so, and the retries slow down', () => {
+    const c = client();
+    let refused = 0;
+    c.on('refused', () => refused++);
+    c.connect();
+    const delays = [];
+    for (let i = 0; i < 6; i++) {
+        FakeSocket.last.drop();
+        delays.push(...c.timers.runAll());
+    }
+    assert.equal(refused, 6);
+    assert.deepEqual(delays, [500, 1000, 2000, 4000, 8000, 10000]);
+});
+
+test('close stops the reconnecting', () => {
+    const { c } = connected();
+    c.close();
+    assert.deepEqual(c.timers.runAll(), []);
+});
+
+test('events update the model, and are told by name', () => {
+    const { c, ws } = connected();
+    const shade = c.model.shadeList()[0];
+    const seen = [];
+    c.on('shadeState', (d) => seen.push(d));
+    ws.deliver({ v: 2, t: T.shadeState, d: { s: shade.shadeId, t: 30, d: 1 } });
+    assert.equal(c.model.shades.get(shade.shadeId).target, 30);
+    assert.deepEqual(seen, [{ shadeId: shade.shadeId, target: 30, direction: 1 }]);
+});
+
+test('an event before the schema waits for it, and goes before the sync frames formatted after it', () => {
+    const c = client();
+    const states = [];
+    c.on('shadeState', (d) => states.push(d));
+    c.connect();
+    const ws = FakeSocket.last;
+    ws.open();
+    ws.deliver(hello);
+    // Before the schema: the client can't read it yet.
+    const recorded = recordedItems('S')[0];
+    ws.deliver({ v: 2, t: T.shadeState, d: { s: recorded.s, t: 55 } });
+    assert.deepEqual(states, []);
+    for (const f of sync) ws.deliver(f);
+    assert.deepEqual(states, [{ shadeId: recorded.s, target: 55 }]);
+    // The shade's sync frame was formatted after the event, so it wins.
+    assert.equal(c.model.shades.get(recorded.s).target, recorded.t || 0);
+});
+
+test('an event after the schema goes at once, during the sync', () => {
+    const c = client();
+    c.connect();
+    const ws = FakeSocket.last;
+    ws.open();
+    ws.deliver(hello);
+    const syncFrames = sync.filter((f) => f.t === T.sync);
+    for (const f of syncFrames) ws.deliver(f);
+    const shadeId = recordedItems('S')[0].s;
+    ws.deliver({ v: 2, t: T.shadeState, d: { s: shadeId, t: 66 } });
+    assert.equal(c.ready, false);
+    assert.equal(c.model.shades.get(shadeId).target, 66);
+});
+
+test('the schema is kept, and the device is told it has it', () => {
+    const storage = new FakeStorage();
+    connected({ storage: storage });
+    const c = client({ storage: storage });
+    c.connect();
+    const ws = FakeSocket.last;
+    assert.equal(ws.url, `ws://device/ws?schema=${hello.d.sh}`);
+    ws.open();
+    ws.deliver(hello);
+    for (const f of syncWithoutSchema()) ws.deliver(f);
+    assert.ok(c.ready);
+    assert.equal(c.model.shades.size, recordedItems('S').length);
+});
+
+test('a schema cut off by a lost socket is not kept, so it comes whole again', () => {
+    const c = client();
+    c.connect();
+    const ws = FakeSocket.last;
+    ws.open();
+    ws.deliver(hello);
+    ws.deliver(sync[0]);
+    ws.drop();
+    c.timers.runAll();
+    assert.equal(FakeSocket.last.url, 'ws://device/ws');
+    FakeSocket.last.open();
+    FakeSocket.last.deliver(hello);
+    for (const f of sync) FakeSocket.last.deliver(f);
+    assert.equal(Object.keys(c.schema.fields).length, Object.keys(schema.fields).length);
+    assert.equal(c.schema.hash, hello.d.sh);
+});
+
+test('a socket is replaced, never doubled', () => {
+    const { c, ws } = connected();
+    c.connect();
+    assert.equal(FakeSocket.count, 1);
+    // A new token while a reconnect is waiting: one socket, and no timer left.
+    ws.drop();
+    c.setToken('new');
+    assert.equal(FakeSocket.count, 2);
+    assert.deepEqual(c.timers.runAll(), []);
+    // While it connects: the old one goes without a refused.
+    let refused = 0;
+    c.on('refused', () => refused++);
+    c.setToken('newer');
+    assert.equal(refused, 0);
+    assert.equal(FakeSocket.count, 3);
+    assert.match(FakeSocket.last.url, /apikey=newer/);
+});
+
+test('a listener can stop the retries after a refusal', () => {
+    const c = client();
+    c.on('refused', () => c.close());
+    c.connect();
+    FakeSocket.last.drop();
+    assert.deepEqual(c.timers.runAll(), []);
+    assert.equal(FakeSocket.count, 1);
+});
+
+test('feeds are subscribed again after a reconnect', () => {
+    const { c, ws } = connected();
+    // Its reply never comes: the socket closes first.
+    c.subscribe('frames').catch(() => {});
+    ws.drop();
+    c.timers.runAll();
+    const again = FakeSocket.last;
+    again.open();
+    again.deliver(hello);
+    for (const f of sync) again.deliver(f);
+    const sent = again.sent.filter((f) => f.t === T.subscribe);
+    assert.deepEqual(sent.map((f) => f.d), [{ fe: schema.feedNumbers.frames }]);
+});
+
+test('a feed the device lacks is refused here', async () => {
+    const { c } = connected();
+    await assert.rejects(c.subscribe('nope'), (err) => err.code === 'badRequest');
+});
+
+test('a long op refused as busy without its id goes again', () => {
+    const { c, ws } = connected();
+    c.op('saveRoom', { roomId: 1, name: 'x'.repeat(1500) }).catch(() => {});
+    const sent = ws.sent.length;
+    ws.deliver(reply(undefined, { e: schema.errorNumbers.busy, x: 'Another long message is on its way' }));
+    c.timers.runAll();
+    assert.equal(ws.sent.length, sent + 1);
+    assert.equal(ws.sent[ws.sent.length - 1].d.n.length, 1500);
+});
+
+test('a refusal without an id for nothing long is reported', () => {
+    const { c, ws } = connected();
+    let error = null;
+    c.on('error', (err) => { error = err; });
+    ws.deliver(reply(undefined, { e: schema.errorNumbers.badRequest, x: 'Malformed message' }));
+    assert.equal(error.code, 'badRequest');
+});
+
+test('a sync asked for and refused is called off', async () => {
+    const { c, ws } = connected();
+    const p = c.resync();
+    const id = ws.sent[ws.sent.length - 1].id;
+    ws.deliver(reply(id, { e: schema.errorNumbers.busy, x: 'busy' }));
+    await assert.rejects(p);
+    assert.equal(c.model.seen, null);
+});
+
+test('a kept schema of another firmware is dropped', () => {
+    const storage = new FakeStorage();
+    connected({ storage: storage });
+    const c = client({ storage: storage });
+    c.connect();
+    const ws = FakeSocket.last;
+    ws.open();
+    ws.deliver({ v: 2, t: 1, d: { fw: 'v4.0.0', sh: 'ffffffff' } });
+    // Its hash only counts once the new one has come whole.
+    assert.equal(c.schema.hash, null);
+    assert.ok(c.schema.empty);
+});
+
+test('a frame of another protocol version is reported', () => {
+    const { c, ws } = connected();
+    let mismatch = null;
+    c.on('versionMismatch', (d) => { mismatch = d; });
+    ws.deliver({ v: 3, t: 1, d: {} });
+    assert.deepEqual(mismatch, { device: 3, client: 2 });
+});
+
+test('a new token connects again with it', () => {
+    const { c, ws } = connected();
+    c.setToken('new');
+    assert.ok(ws.closed);
+    c.timers.runAll();
+    assert.match(FakeSocket.last.url, /apikey=new/);
+});
+
+test('waitFor resolves with the first event that matches, after the model has it', async () => {
+    const { c, ws } = connected();
+    const shadeId = recordedItems('S')[0].s;
+    const p = c.waitFor('shade', (d) => d.shadeId === shadeId, 3000);
+    ws.deliver({ v: 2, t: T.shade, d: { s: 250, n: 'Other' } });
+    ws.deliver({ v: 2, t: T.shade, d: { s: shadeId, n: 'Renamed' } });
+    const d = await p;
+    assert.equal(d.name, 'Renamed');
+    assert.equal(c.model.shades.get(shadeId).name, 'Renamed');
+});
+
+test('waitFor gives null when no event comes, and stops listening', async () => {
+    const { c } = connected();
+    const p = c.waitFor('room', null, 3000);
+    c.timers.runAll();
+    assert.equal(await p, null);
+    assert.equal((c.listeners.room || []).length, 0);
+});
+
+test('off stops a listener, and one that removes itself mid-event leaves the others called', () => {
+    const { c, ws } = connected();
+    const seen = [];
+    const once = (d) => { seen.push('once'); c.off('memory', once); };
+    const always = () => seen.push('always');
+    const dropped = () => seen.push('dropped');
+    c.on('memory', once);
+    c.on('memory', always);
+    c.on('memory', dropped);
+    c.off('memory', dropped);
+    ws.deliver({ v: 2, t: T.memory, d: { fr: 1 } });
+    ws.deliver({ v: 2, t: T.memory, d: { fr: 2 } });
+    assert.deepEqual(seen, ['once', 'always', 'always']);
+});
+
+test('subscribe sends the feed number', () => {
+    const { c, ws } = connected();
+    c.subscribe('frames');
+    const sent = ws.sent[ws.sent.length - 1];
+    assert.equal(sent.t, T.subscribe);
+    assert.deepEqual(sent.d, { fe: schema.feedNumbers.frames });
+});

test/js/helpers.js

@@ -0,0 +1,84 @@
+// What the web UI client tests share: the frames the e2e suite recorded from a
+// device (test/fixtures/proto2), and stand-ins for the socket and the timers.
+const path = require('node:path');
+
+const FIXTURES = path.join(__dirname, '..', 'fixtures', 'proto2');
+const hello = require(path.join(FIXTURES, 'hello.json'));
+const sync = require(path.join(FIXTURES, 'sync.json'));
+const schemaJson = require(path.join(FIXTURES, 'schema.json'));
+const { ProtoSchema } = require('../../data-src/js/schema.js');
+
+// A schema filled from the recorded sync, as a client builds it.
+function recordedSchema() {
+    const s = new ProtoSchema();
+    for (const f of sync) if (f.t === 4) s.add(f.d);
+    s.hash = hello.d.sh;
+    return s;
+}
+// The recorded sync's device sections, expanded, frame by frame.
+function deviceSections(schema) {
+    return sync.filter((f) => f.t === 4).map((f) => schema.expand(ProtoSchema.withoutRows(f.d)));
+}
+// The recorded sync as the device sends it to a client that has the schema.
+function syncWithoutSchema() {
+    return sync.map((f) => Object.assign({}, f, { d: ProtoSchema.withoutRows(f.d || {}) }));
+}
+// The recorded sync's items under their short keys.
+function recordedItems(key) {
+    return sync.filter((f) => f.t === 4 && f.d[key]).flatMap((f) => f.d[key]);
+}
+
+// A WebSocket the test drives: it records what the client sends, and the test
+// opens, delivers and closes.
+class FakeSocket {
+    constructor(url) {
+        this.url = url;
+        this.sent = [];
+        this.closed = false;
+        FakeSocket.last = this;
+        FakeSocket.count++;
+    }
+    send(text) { this.sent.push(JSON.parse(text)); }
+    close() {
+        if (this.closed) return;
+        this.closed = true;
+        if (this.onclose) this.onclose({});
+    }
+    open() { if (this.onopen) this.onopen({}); }
+    deliver(frame) { this.onmessage({ data: typeof frame === 'string' ? frame : JSON.stringify(frame) }); }
+    // The device closing it, or refusing it before it opened.
+    drop() { this.close(); }
+}
+// How many the client has opened.
+FakeSocket.count = 0;
+FakeSocket.reset = () => { FakeSocket.count = 0; FakeSocket.last = null; };
+
+// Timers the test runs by hand.
+class FakeTimers {
+    constructor() {
+        this.timers = new Map();
+        this.next = 1;
+        this.setTimeout = (fn, ms) => {
+            const id = this.next++;
+            this.timers.set(id, { fn: fn, ms: ms });
+            return id;
+        };
+        this.clearTimeout = (id) => { this.timers.delete(id); };
+    }
+    // Runs every timer waiting, the shortest first, and returns their delays.
+    runAll() {
+        const due = [...this.timers.entries()].sort((a, b) => a[1].ms - b[1].ms);
+        this.timers.clear();
+        for (const [, t] of due) t.fn();
+        return due.map(([, t]) => t.ms);
+    }
+}
+// localStorage as far as the client uses it.
+class FakeStorage {
+    constructor() { this.items = {}; }
+    getItem(key) { return Object.prototype.hasOwnProperty.call(this.items, key) ? this.items[key] : null; }
+    setItem(key, value) { this.items[key] = String(value); }
+}
+
+module.exports = { hello, sync, schemaJson, recordedSchema, deviceSections, syncWithoutSchema, recordedItems, FakeSocket,
+    FakeTimers, FakeStorage };

test/js/model.test.js

@@ -0,0 +1,125 @@
+// The web UI's ProtoModel (data-src/js/model.js): the recorded sync, then
+// events.
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const { ProtoModel } = require('../../data-src/js/model.js');
+const { recordedSchema, deviceSections, recordedItems } = require('./helpers.js');
+
+function synced() {
+    const schema = recordedSchema();
+    const model = new ProtoModel(schema);
+    model.beginSync();
+    for (const d of deviceSections(schema)) model.applySync(d);
+    model.endSync();
+    return model;
+}
+
+test('the sync fills every room, shade and group', () => {
+    const m = synced();
+    assert.equal(m.rooms.size, recordedItems('R').length);
+    assert.equal(m.shades.size, recordedItems('S').length);
+    assert.equal(m.groups.size, recordedItems('G').length);
+    assert.equal(m.header.model, 'ESPSomfyRTS');
+    assert.equal(m.header.maxShades, 32);
+    assert.ok(m.status.freeHeap > 0);
+});
+
+test('a field the sync left out is its default', () => {
+    const m = synced();
+    const raw = recordedItems('S').find((s) => typeof s.m === 'undefined');
+    const shade = m.shades.get(raw.s);
+    // Left out because they were at their default.
+    assert.equal(shade.myPos, -1);
+    assert.equal(shade.stepSize, raw.z === undefined ? 100 : raw.z);
+    assert.equal(typeof shade.paired, 'boolean');
+    assert.ok(Array.isArray(shade.linkedRemotes));
+    for (const r of shade.linkedRemotes) assert.equal(typeof r.lastRollingCode, 'number');
+    for (const g of m.groupList()) assert.ok(Array.isArray(g.linkedShades));
+});
+
+test('the lists come in their sort order', () => {
+    const m = synced();
+    const orders = m.shadeList().map((s) => s.sortOrder);
+    assert.deepEqual(orders, [...orders].sort((a, b) => a - b));
+});
+
+test('a state event changes only the state, and a left-out field is its default', () => {
+    const m = synced();
+    const shade = m.shadeList()[0];
+    const name = shade.name;
+    // A shade at rest at the top: position 0 is left out.
+    assert.ok(m.applyEvent('shadeState', { shadeId: shade.shadeId, target: 40, direction: 1 }));
+    assert.equal(m.shades.get(shade.shadeId).position, 0);
+    assert.equal(m.shades.get(shade.shadeId).target, 40);
+    assert.equal(m.shades.get(shade.shadeId).name, name);
+    // Of a shade the client doesn't have yet: nothing to change.
+    assert.equal(m.applyEvent('shadeState', { shadeId: 250, position: 5 }), false);
+});
+
+test('a whole item replaces what was there', () => {
+    const m = synced();
+    const shade = m.shadeList()[0];
+    m.applyEvent('shade', { shadeId: shade.shadeId, name: 'Renamed' });
+    assert.equal(m.shades.get(shade.shadeId).name, 'Renamed');
+    assert.equal(m.shades.get(shade.shadeId).upTime, 0);
+    m.applyEvent('room', { roomId: 9, name: 'New' });
+    assert.equal(m.rooms.get(9).sortOrder, 0);
+});
+
+test('removals, orders and repeaters', () => {
+    const m = synced();
+    const changes = [];
+    m.onChange((kind, id) => changes.push([kind, id]));
+    const [first, second] = m.shadeList();
+    assert.ok(m.applyEvent('shadeRemoved', { shadeId: first.shadeId }));
+    assert.equal(m.shades.has(first.shadeId), false);
+    // An id it doesn't have is ignored.
+    assert.equal(m.applyEvent('roomRemoved', { roomId: 250 }), false);
+    m.applyEvent('shadeOrder', { ids: [second.shadeId] });
+    assert.equal(m.shades.get(second.shadeId).sortOrder, 0);
+    m.applyEvent('repeaters', { repeaters: [7, 8] });
+    assert.deepEqual(m.repeaters, [7, 8]);
+    // An order says which list it is.
+    assert.deepEqual(changes.map((c) => c[0]), ['shadeRemoved', 'shadeOrder', 'repeaters']);
+});
+
+test('a sync without repeaters clears them, as the device leaves the list out when it has none', () => {
+    const m = synced();
+    m.applyEvent('repeaters', { repeaters: [7] });
+    m.beginSync();
+    for (const d of deviceSections(m.schema)) m.applySync(Object.assign({}, d, { repeaters: undefined }));
+    m.endSync();
+    assert.deepEqual(m.repeaters, []);
+});
+
+test('a sync that is called off removes nothing', () => {
+    const m = synced();
+    const shades = m.shades.size;
+    m.beginSync();
+    m.cancelSync();
+    m.endSync();
+    assert.equal(m.shades.size, shades);
+});
+
+test('a sync drops what it no longer lists, and keeps what an event added during it', () => {
+    const m = synced();
+    const schema = m.schema;
+    m.applyEvent('room', { roomId: 99, name: 'Gone' });
+    m.beginSync();
+    m.applyEvent('room', { roomId: 98, name: 'Added during the sync' });
+    for (const d of deviceSections(schema)) m.applySync(d);
+    m.endSync();
+    assert.equal(m.rooms.has(99), false);
+    assert.equal(m.rooms.has(98), true);
+});
+
+test('status events', () => {
+    const m = synced();
+    m.applyEvent('memory', { freeHeap: 1000, uptime: 5 });
+    assert.equal(m.status.freeHeap, 1000);
+    assert.equal(m.status.loopRate, 0);
+    m.applyEvent('wifi', { strength: -60, channel: 6 });
+    assert.equal(m.status.ssid, '');
+    assert.equal(m.status.strength, -60);
+    assert.equal(m.applyEvent('frame', { remoteAddress: 1 }), false);
+});

test/js/schema.test.js

@@ -0,0 +1,56 @@
+// The web UI's ProtoSchema (data-src/js/schema.js) against the schema a
+// device sent (test/fixtures/proto2).
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const { ProtoSchema } = require('../../data-src/js/schema.js');
+const { recordedSchema, schemaJson } = require('./helpers.js');
+
+test('the schema is collected from every sync frame it spans', () => {
+    const s = recordedSchema();
+    assert.equal(Object.keys(s.types).length, Object.keys(schemaJson.types).length);
+    assert.equal(Object.keys(s.fields).length, Object.keys(schemaJson.fields).length);
+    assert.equal(s.typeNumbers.shadeCommand, 16);
+    assert.equal(s.typeNumbers.saveSettings, 50);
+    assert.equal(s.keys.shadeId, 's');
+    assert.equal(s.errorNumbers.busy, 5);
+    assert.equal(s.feedNumbers.frequencyScan, 2);
+});
+
+test('short keys become names, and 1/0 booleans, all the way down', () => {
+    const s = recordedSchema();
+    const shade = s.expand({ s: 3, n: 'Office', i: 1, fp: 0, lk: [{ a: 5, c: 9 }] });
+    assert.deepEqual(shade, { shadeId: 3, name: 'Office', paired: true, flipPosition: false,
+        linkedRemotes: [{ remoteAddress: 5, lastRollingCode: 9 }] });
+    // A whole number that happens to be 1 stays a number.
+    assert.deepEqual(s.expand({ p: 1 }), { position: 1 });
+});
+
+test('names become short keys for an op', () => {
+    const s = recordedSchema();
+    assert.deepEqual(s.pack({ shadeId: 3, command: 'Up', paired: true, sunny: false }), { s: 3, cm: 'Up', i: 1, sn: 0 });
+    assert.deepEqual(s.pack({ ids: [3, 1, 2] }), { is: [3, 1, 2] });
+    // Left out, not sent as undefined.
+    assert.deepEqual(s.pack({ shadeId: 3, target: undefined }), { s: 3 });
+    assert.throws(() => s.pack({ nonsense: 1 }), /no field nonsense/);
+});
+
+test('a field left out is its default', () => {
+    const s = recordedSchema();
+    assert.equal(s.defaultOf('myPos'), -1);
+    assert.equal(s.defaultOf('stepSize'), 100);
+    assert.equal(s.defaultOf('bitLength'), 56);
+    assert.equal(s.defaultOf('paired'), false);
+    assert.equal(s.defaultOf('name'), '');
+    assert.deepEqual(s.defaultOf('linkedShades'), []);
+    assert.deepEqual(s.fill({ shadeId: 2 }, ['shadeId', 'position', 'myPos']), { shadeId: 2, position: 0, myPos: -1 });
+});
+
+test('a cached schema comes back whole, with its hash', () => {
+    const s = recordedSchema();
+    const back = ProtoSchema.load(JSON.parse(JSON.stringify(s.save())));
+    assert.equal(back.hash, s.hash);
+    assert.deepEqual(back.fields, s.fields);
+    assert.deepEqual(back.types, s.types);
+    assert.equal(ProtoSchema.load(null), null);
+    assert.equal(ProtoSchema.load({ hash: 'x', rows: {} }), null);
+});

test/js/views.test.js

@@ -0,0 +1,82 @@
+// What the web UI's panels bind (data-src/js/views.js), from protocol 2's
+// shapes.
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const { ProtoViews } = require('../../data-src/js/views.js');
+const { ProtoModel } = require('../../data-src/js/model.js');
+const { recordedSchema, deviceSections } = require('./helpers.js');
+
+const schema = recordedSchema();
+function synced() {
+    const model = new ProtoModel(schema);
+    model.beginSync();
+    for (const d of deviceSections(schema)) model.applySync(d);
+    model.endSync();
+    return model;
+}
+
+test('a settings section has every field, defaults where the device left them out', () => {
+    // ssdpBroadcast off and no NTP server are at their defaults, so not sent.
+    assert.deepEqual(ProtoViews.settings(schema, 'general', { hostname: 'Somfy' }),
+        { hostname: 'Somfy', ssdpBroadcast: false, ntpServer: '', posixZone: '' });
+    assert.deepEqual(ProtoViews.settings(schema, 'security', {}), { securityType: 0, username: '', hasPassword: false, hasPin: false });
+});
+
+test('the settings sections are what the device sends', () => {
+    for (const section of Object.keys(ProtoViews.SETTINGS))
+        for (const name of ProtoViews.SETTINGS[section]) assert.ok(schema.keys[name], `${section}.${name}`);
+});
+
+test('a group shows its shades whole', () => {
+    const m = synced();
+    const group = m.groupList()[0];
+    const view = ProtoViews.group(m, group);
+    assert.equal(view.linkedShades.length, group.linkedShades.length);
+    assert.equal(view.linkedShades[0].name, m.shades.get(group.linkedShades[0]).name);
+    // The model's item is left as it is.
+    assert.equal(typeof group.linkedShades[0], 'number');
+    // A shade the model hasn't got yet still shows.
+    assert.equal(ProtoViews.group(m, Object.assign({}, group, { linkedShades: [250] })).linkedShades[0].shadeId, 250);
+    assert.equal(ProtoViews.group(m, undefined), undefined);
+});
+
+test('a frame and a scan under the names the UI reads', () => {
+    const f = ProtoViews.frame(schema, { remoteAddress: 5, lastRollingCode: 9, command: 'Up', rssi: -60, valid: true, pulses: [1, 2] });
+    assert.equal(f.address, 5);
+    assert.equal(f.rcode, 9);
+    assert.equal(f.encKey, 0);
+    assert.deepEqual(f.pulses, [1, 2]);
+    assert.equal(f.stepSize, undefined);
+    const s = ProtoViews.scan(schema, { scanning: true, testFreq: 433.1, testRssi: -80 });
+    assert.deepEqual(s, { scanning: true, testFreq: 433.1, testRSSI: -80, frequency: 0, RSSI: 0 });
+});
+
+test('the network section as the panel binds it', () => {
+    const v = ProtoViews.network(schema, { connMode: 1, ssid: 'home', hasPassphrase: true, dhcp: true, PWRPin: -1 });
+    assert.equal(v.connType, 1);
+    assert.equal(v.hasEthernet, false);
+    assert.deepEqual(v.wifi, { ssid: 'home', hasPassphrase: true, roaming: false, hidden: false });
+    assert.equal(v.ethernet.PWRPin, -1);
+    assert.deepEqual(v.ip, { dhcp: true, ip: '', gateway: '', subnet: '', dns1: '', dns2: '' });
+});
+
+test('the radio section as the panel binds it, and back', () => {
+    const config = ProtoViews.radio(schema, { bitLength: 80, frequency: 433.42, enabled: true, radioInit: true });
+    assert.equal(config.type, 80);
+    assert.equal(config.bitLength, undefined);
+    assert.equal(config.TXPin, 0);
+    const save = ProtoViews.radioSave(config);
+    assert.equal(save.section, 'radio');
+    assert.equal(save.bitLength, 80);
+    assert.equal(save.frequency, 433.42);
+    // Only shown, never saved.
+    assert.equal('radioInit' in save, false);
+    assert.equal('type' in save, false);
+});
+
+test('an op refusal for the error box, with a shade validation code', () => {
+    assert.deepEqual(ProtoViews.opError('saveShade', { code: 'badRequest', desc: 'Pin in use', data: { validation: -10 } }),
+        { htmlError: 'badRequest', service: 'saveShade', desc: 'Pin in use', code: -10 });
+    assert.deepEqual(ProtoViews.opError('reboot', { code: 'closed', desc: 'The connection closed' }),
+        { htmlError: 'closed', service: 'reboot', desc: 'The connection closed' });
+});

test/test_loopqueue/test_loopqueue.cpp

@@ -21,7 +21,9 @@ static const shade_request_type_t OFF_RADIO_REQUESTS[] = {
   shade_request_type_t::set_positions, shade_request_type_t::set_rolling_code
 };
 static const loop_item_type_t OTHER_ITEMS[] = {
-  loop_item_type_t::restore_shades, loop_item_type_t::load_shades, loop_item_type_t::model_edit, loop_item_type_t::proto_read
+  loop_item_type_t::restore_shades, loop_item_type_t::load_shades, loop_item_type_t::model_edit, loop_item_type_t::proto_read,
+  // A frequency scan's end among them, so it never waits for the radio.
+  loop_item_type_t::proto_device
 };
 
 void setUp() {}

test/test_modeleditor/test_modeleditor.cpp

@@ -205,16 +205,30 @@ static void test_a_repeater_edit_sends_the_repeaters() {
   TEST_ASSERT_TRUE(editChanges(model_edit_type_t::unlink_repeater, resultFor(0)).repeaters);
   TEST_ASSERT_FALSE(editChanges(model_edit_type_t::save_room, resultFor(1)).repeaters);
 }
-static void test_deletes_send_nothing_more() {
-  // Their removals go out as they happen.
-  for(model_edit_type_t t : {model_edit_type_t::delete_room, model_edit_type_t::delete_shade, model_edit_type_t::delete_group}) {
-    const edit_changes_t c = editChanges(t, resultFor(4));
+static void test_deletes_send_their_removal() {
+  // After the reply, as every edit's events are, so a client waiting for the
+  // reply doesn't read past it.
+  const edit_changes_t room = editChanges(model_edit_type_t::delete_room, resultFor(4));
+  const edit_changes_t shade = editChanges(model_edit_type_t::delete_shade, resultFor(5));
+  const edit_changes_t group = editChanges(model_edit_type_t::delete_group, resultFor(6));
+  TEST_ASSERT_EQUAL_UINT8(4, room.removedRoomId);
+  TEST_ASSERT_EQUAL_UINT8(5, shade.removedShadeId);
+  TEST_ASSERT_EQUAL_UINT8(6, group.removedGroupId);
+  for(const edit_changes_t &c : {room, shade, group}) {
+    // Nothing is left of it to send whole.
     TEST_ASSERT_EQUAL_UINT8(0, c.shadeId);
     TEST_ASSERT_EQUAL_UINT8(0, c.groupId);
     TEST_ASSERT_EQUAL_UINT8(0, c.roomId);
     TEST_ASSERT_EQUAL(sort_list_t::none, c.order);
     TEST_ASSERT_FALSE(c.repeaters);
   }
+  TEST_ASSERT_EQUAL_UINT8(0, room.removedShadeId + room.removedGroupId);
+  TEST_ASSERT_EQUAL_UINT8(0, shade.removedRoomId + shade.removedGroupId);
+  TEST_ASSERT_EQUAL_UINT8(0, group.removedRoomId + group.removedShadeId);
+}
+static void test_other_edits_remove_nothing() {
+  const edit_changes_t c = editChanges(model_edit_type_t::save_room, resultFor(4));
+  TEST_ASSERT_EQUAL_UINT8(0, c.removedRoomId + c.removedShadeId + c.removedGroupId);
 }
 
 int main(int argc, char **argv) {
@@ -249,6 +263,7 @@ int main(int argc, char **argv) {
   RUN_TEST(test_a_room_edit_sends_the_room);
   RUN_TEST(test_a_sort_edit_sends_its_list);
   RUN_TEST(test_a_repeater_edit_sends_the_repeaters);
-  RUN_TEST(test_deletes_send_nothing_more);
+  RUN_TEST(test_deletes_send_their_removal);
+  RUN_TEST(test_other_edits_remove_nothing);
   return UNITY_END();
 }

test/test_proto/test_proto.cpp

@@ -60,6 +60,12 @@ static void test_every_op_says_what_becomes_of_it() {
   TEST_ASSERT_EQUAL(proto_op_t::remoteFrame, protoOpKind(static_cast<uint8_t>(proto_type_t::sendRemoteCommand)));
   TEST_ASSERT_EQUAL(proto_op_t::edit, protoOpKind(static_cast<uint8_t>(proto_type_t::saveShade)));
   TEST_ASSERT_EQUAL(proto_op_t::read, protoOpKind(static_cast<uint8_t>(proto_type_t::groupOptions)));
+  TEST_ASSERT_EQUAL(proto_op_t::device, protoOpKind(static_cast<uint8_t>(proto_type_t::getSettings)));
+  TEST_ASSERT_EQUAL(proto_op_t::device, protoOpKind(static_cast<uint8_t>(proto_type_t::saveSettings)));
+  TEST_ASSERT_EQUAL(proto_op_t::device, protoOpKind(static_cast<uint8_t>(proto_type_t::scanWifi)));
+  TEST_ASSERT_EQUAL(proto_op_t::device, protoOpKind(static_cast<uint8_t>(proto_type_t::beginFrequencyScan)));
+  TEST_ASSERT_EQUAL(proto_op_t::device, protoOpKind(static_cast<uint8_t>(proto_type_t::endFrequencyScan)));
+  TEST_ASSERT_EQUAL(proto_op_t::device, protoOpKind(static_cast<uint8_t>(proto_type_t::reboot)));
   TEST_ASSERT_EQUAL(proto_op_t::none, protoOpKind(250));
 }
 static void test_a_field_by_its_key() {

test/test_protosettings/test_protosettings.cpp

@@ -0,0 +1,223 @@
+#include <unity.h>
+#include <string>
+#include <ArduinoJson.h>
+#include "ProtoSettings.h"
+
+// Socket protocol 2's settings ops: the sections, and the settings route's
+// request that a save's payload becomes (docs/PROTOCOL.md).
+
+void setUp() {}
+void tearDown() {}
+
+static JsonDocument payload;
+static JsonDocument doc;
+static settings_section_t section;
+
+static void load(const char *json) {
+  payload.clear();
+  TEST_ASSERT_EQUAL(DeserializationError::Ok, deserializeJson(payload, json).code());
+}
+static const char *expand(const char *json, bool hasEthernet = true) {
+  load(json);
+  return expandSettingsPayload(payload.as<JsonObjectConst>(), hasEthernet, section, doc);
+}
+static const char *sectionOf(const char *json) {
+  load(json);
+  return parseSettingsSection(payload.as<JsonObjectConst>(), section);
+}
+static std::string written() {
+  std::string out;
+  serializeJson(doc, out);
+  return out;
+}
+static uint16_t parts() { return settingsParts(section, doc.as<JsonObjectConst>()); }
+static uint16_t bit(settings_part_t part) { return 1u << static_cast<uint8_t>(part); }
+#define TEST_ASSERT_GOOD(why) TEST_ASSERT_NULL_MESSAGE(why, why)
+
+/*********************************************************************
+ * Sections
+ ********************************************************************/
+static void test_every_section_by_its_name() {
+  const settings_section_t all[] = {settings_section_t::general, settings_section_t::network, settings_section_t::mqtt,
+    settings_section_t::security, settings_section_t::radio};
+  for(settings_section_t s : all) {
+    std::string json = std::string(R"({"se":")") + settingsSectionName(s) + "\"}";
+    TEST_ASSERT_GOOD(sectionOf(json.c_str()));
+    TEST_ASSERT_EQUAL(s, section);
+  }
+  TEST_ASSERT_EQUAL_STRING("general", settingsSectionName(settings_section_t::general));
+  TEST_ASSERT_EQUAL_STRING("radio", settingsSectionName(settings_section_t::radio));
+}
+static void test_a_section_must_be_named() {
+  TEST_ASSERT_EQUAL_STRING("section is missing", sectionOf(R"({})"));
+  TEST_ASSERT_EQUAL_STRING("section is missing", sectionOf(R"({"se":2})"));
+  TEST_ASSERT_EQUAL_STRING("Unknown section", sectionOf(R"({"se":"wifi"})"));
+  // Names are matched as they are.
+  TEST_ASSERT_EQUAL_STRING("Unknown section", sectionOf(R"({"se":"General"})"));
+  TEST_ASSERT_EQUAL_STRING("section is missing", expand(R"({"hn":"x"})"));
+}
+
+/*********************************************************************
+ * A save's payload as the route's request
+ ********************************************************************/
+static void test_short_keys_become_the_route_names() {
+  TEST_ASSERT_GOOD(expand(R"({"se":"general","hn":"Somfy","ns":"pool.ntp.org"})"));
+  TEST_ASSERT_EQUAL(settings_section_t::general, section);
+  TEST_ASSERT_EQUAL_STRING(R"({"hostname":"Somfy","ntpServer":"pool.ntp.org"})", written().c_str());
+}
+static void test_fields_named_apart_from_the_dictionary() {
+  // connType in the header is the connection in use; the setting is connMode.
+  TEST_ASSERT_GOOD(expand(R"({"se":"network","cn":2})"));
+  TEST_ASSERT_EQUAL_STRING(R"({"connType":2})", written().c_str());
+  TEST_ASSERT_GOOD(expand(R"({"se":"security","sy":1,"pn":"1234"})"));
+  TEST_ASSERT_EQUAL_STRING(R"({"type":1,"pin":"1234"})", written().c_str());
+  TEST_ASSERT_GOOD(expand(R"({"se":"radio","bl":80})"));
+  TEST_ASSERT_EQUAL_STRING(R"({"type":80})", written().c_str());
+}
+static void test_booleans_become_true_and_false() {
+  TEST_ASSERT_GOOD(expand(R"({"se":"mqtt","en":1,"pd":0})"));
+  TEST_ASSERT_EQUAL_STRING(R"({"enabled":true,"pubDisco":false})", written().c_str());
+  TEST_ASSERT_EQUAL_STRING("enabled is not a boolean (1 or 0)", expand(R"({"se":"radio","en":true})"));
+}
+static void test_values_are_checked_against_their_kind() {
+  TEST_ASSERT_EQUAL_STRING("port is not a whole number", expand(R"({"se":"mqtt","po":"1883"})"));
+  TEST_ASSERT_EQUAL_STRING("hostname is not a string", expand(R"({"se":"general","hn":5})"));
+  TEST_ASSERT_EQUAL_STRING("frequency is not a number", expand(R"({"se":"radio","fq":"433"})"));
+}
+static void test_a_number_may_be_whole() {
+  TEST_ASSERT_GOOD(expand(R"({"se":"radio","fq":433,"bw":99.97})"));
+  TEST_ASSERT_EQUAL_STRING(R"({"frequency":433,"rxBandwidth":99.97})", written().c_str());
+}
+static void test_whole_numbers_are_checked_against_what_the_setting_holds() {
+  TEST_ASSERT_EQUAL_STRING("connMode must be 1-3", expand(R"({"se":"network","cn":4})"));
+  TEST_ASSERT_EQUAL_STRING("connMode must be 1-3", expand(R"({"se":"network","cn":0})"));
+  TEST_ASSERT_EQUAL_STRING("port must be 1-65535", expand(R"({"se":"mqtt","po":65536})"));
+  TEST_ASSERT_EQUAL_STRING("txPower must be -30-12", expand(R"({"se":"radio","tp":13})"));
+  TEST_ASSERT_EQUAL_STRING("TXPin must be 0-48", expand(R"({"se":"radio","tx":-1})"));
+  TEST_ASSERT_EQUAL_STRING("securityType must be 0-2", expand(R"({"se":"security","sy":3})"));
+  TEST_ASSERT_EQUAL_STRING("port must be 1-65535", expand(R"({"se":"mqtt","po":18446744073709551615})"));
+  // A pin of -1 is none.
+  TEST_ASSERT_GOOD(expand(R"({"se":"network","pw":-1})"));
+  TEST_ASSERT_GOOD(expand(R"({"se":"radio","tp":-30})"));
+}
+static void test_the_bit_length_is_56_or_80() {
+  TEST_ASSERT_GOOD(expand(R"({"se":"radio","bl":56})"));
+  TEST_ASSERT_EQUAL_STRING("bitLength must be 56 or 80", expand(R"({"se":"radio","bl":60})"));
+  TEST_ASSERT_EQUAL_STRING("bitLength must be 56 or 80", expand(R"({"se":"radio","bl":0})"));
+}
+static void test_the_radio_proto_is_one_there_is() {
+  TEST_ASSERT_GOOD(expand(R"({"se":"radio","pr":2})"));
+  TEST_ASSERT_GOOD(expand(R"({"se":"radio","pr":8})"));
+  TEST_ASSERT_EQUAL_STRING("proto must be 0, 1, 2, 8 or 9", expand(R"({"se":"radio","pr":5})"));
+  TEST_ASSERT_EQUAL_STRING("proto must be 0, 1, 2, 8 or 9", expand(R"({"se":"radio","pr":10})"));
+}
+static void test_numbers_are_checked_against_what_the_radio_can_be_set_to() {
+  // The ends themselves, though 58.03 is a little under it as a float.
+  TEST_ASSERT_GOOD(expand(R"({"se":"radio","fq":300,"bw":58.03,"dv":380.85})"));
+  TEST_ASSERT_GOOD(expand(R"({"se":"radio","fq":928,"bw":812.5,"dv":1.58})"));
+  TEST_ASSERT_EQUAL_STRING("frequency must be 300.00-928.00", expand(R"({"se":"radio","fq":299.9})"));
+  TEST_ASSERT_EQUAL_STRING("rxBandwidth must be 58.03-812.50", expand(R"({"se":"radio","bw":900})"));
+  TEST_ASSERT_EQUAL_STRING("deviation must be 1.58-380.85", expand(R"({"se":"radio","dv":0})"));
+}
+static void test_strings_must_fit_their_setting() {
+  // hostname holds 31 characters, a PIN 4.
+  TEST_ASSERT_GOOD(expand(R"({"se":"general","hn":"0123456789012345678901234567890"})"));
+  TEST_ASSERT_EQUAL_STRING("hostname is longer than 31 characters", expand(R"({"se":"general","hn":"01234567890123456789012345678901"})"));
+  TEST_ASSERT_EQUAL_STRING("pin is longer than 4 characters", expand(R"({"se":"security","pn":"12345"})"));
+  // The broker's hostname is another setting, with room for 64.
+  TEST_ASSERT_GOOD(expand(R"({"se":"mqtt","hn":"01234567890123456789012345678901"})"));
+}
+static void test_addresses_are_checked_and_empty_clears_them() {
+  TEST_ASSERT_GOOD(expand(R"({"se":"network","dh":0,"ip":"192.168.1.20","gw":"192.168.1.1","su":"255.255.255.0","d1":"","d2":""})"));
+  TEST_ASSERT_EQUAL_STRING(R"({"dhcp":false,"ip":"192.168.1.20","gateway":"192.168.1.1","subnet":"255.255.255.0","dns1":"0.0.0.0","dns2":"0.0.0.0"})", written().c_str());
+  TEST_ASSERT_EQUAL_STRING("ip is not an IP address", expand(R"({"se":"network","ip":"192.168.1"})"));
+  TEST_ASSERT_EQUAL_STRING("gateway is not an IP address", expand(R"({"se":"network","gw":"192.168.1.256"})"));
+  TEST_ASSERT_EQUAL_STRING("dns1 is not an IP address", expand(R"({"se":"network","d1":"1.2.3.4.5"})"));
+  TEST_ASSERT_EQUAL_STRING("dns2 is not an IP address", expand(R"({"se":"network","d2":"a.b.c.d"})"));
+  TEST_ASSERT_EQUAL_STRING("ip is not an IP address", expand(R"({"se":"network","ip":"1..2.3"})"));
+  TEST_ASSERT_EQUAL_STRING("ip is longer than 15 characters", expand(R"({"se":"network","ip":"0001.0002.0003.0004"})"));
+}
+static void test_only_what_was_sent_is_in_the_request() {
+  TEST_ASSERT_GOOD(expand(R"({"se":"network","ro":0})"));
+  TEST_ASSERT_EQUAL_STRING(R"({"roaming":false})", written().c_str());
+}
+static void test_a_save_needs_something_to_save() {
+  TEST_ASSERT_EQUAL_STRING("Nothing to save", expand(R"({"se":"mqtt"})"));
+}
+static void test_an_unknown_key_is_refused() {
+  TEST_ASSERT_EQUAL_STRING("Unknown field zz", expand(R"({"se":"general","zz":1})"));
+}
+static void test_what_is_only_sent_is_refused() {
+  // Secrets are only said to be set; a save sends the secret itself.
+  TEST_ASSERT_EQUAL_STRING("hasPassword can't be saved in mqtt", expand(R"({"se":"mqtt","hw":1})"));
+  TEST_ASSERT_EQUAL_STRING("hasPin can't be saved in security", expand(R"({"se":"security","hi":1})"));
+  TEST_ASSERT_EQUAL_STRING("hasPassphrase can't be saved in network", expand(R"({"se":"network","hq":1})"));
+  TEST_ASSERT_EQUAL_STRING("hasEthernet can't be saved in network", expand(R"({"se":"network","he":1})"));
+  TEST_ASSERT_EQUAL_STRING("radioInit can't be saved in radio", expand(R"({"se":"radio","ri":1})"));
+}
+static void test_a_field_of_another_section_is_refused() {
+  TEST_ASSERT_EQUAL_STRING("ssid can't be saved in general", expand(R"({"se":"general","sd":"home"})"));
+  TEST_ASSERT_EQUAL_STRING("ntpServer can't be saved in network", expand(R"({"se":"network","ns":"pool.ntp.org"})"));
+  TEST_ASSERT_EQUAL_STRING("TXPin can't be saved in mqtt", expand(R"({"se":"mqtt","tx":12})"));
+  TEST_ASSERT_EQUAL_STRING("shadeId can't be saved in radio", expand(R"({"se":"radio","s":1})"));
+}
+static void test_secure_config_only_is_not_a_field() {
+  // Two modes, None and Secured (ADR 0006): "Secure Config Only" has no key.
+  TEST_ASSERT_EQUAL_STRING("Unknown field pm", expand(R"({"se":"security","pm":1})"));
+}
+static void test_ethernet_needs_a_chip_with_it() {
+  TEST_ASSERT_GOOD(expand(R"({"se":"network","bo":1})"));
+  TEST_ASSERT_EQUAL_STRING("This device has no Ethernet", expand(R"({"se":"network","bo":1})", false));
+  // The rest of the section saves without it.
+  TEST_ASSERT_GOOD(expand(R"({"se":"network","sd":"home","pp":"secret"})", false));
+}
+
+/*********************************************************************
+ * The parts a save sets
+ ********************************************************************/
+static void test_a_save_sets_the_parts_it_has_fields_of() {
+  TEST_ASSERT_GOOD(expand(R"({"se":"network","sd":"home","dh":1})"));
+  TEST_ASSERT_EQUAL_UINT16(bit(settings_part_t::wifi) | bit(settings_part_t::ip), parts());
+  TEST_ASSERT_TRUE(settingsSets(parts(), settings_part_t::wifi));
+  TEST_ASSERT_FALSE(settingsSets(parts(), settings_part_t::conn));
+  TEST_ASSERT_GOOD(expand(R"({"se":"network","cn":3,"ck":1})"));
+  TEST_ASSERT_EQUAL_UINT16(bit(settings_part_t::conn) | bit(settings_part_t::ethernet), parts());
+  TEST_ASSERT_GOOD(expand(R"({"se":"general","sb":1,"pz":"CET-1CEST"})"));
+  TEST_ASSERT_EQUAL_UINT16(bit(settings_part_t::general) | bit(settings_part_t::ntp), parts());
+}
+static void test_a_route_name_belongs_to_its_section() {
+  // hostname is the broker's in mqtt, and type the radio's bit length in radio.
+  TEST_ASSERT_GOOD(expand(R"({"se":"mqtt","hn":"broker"})"));
+  TEST_ASSERT_EQUAL_UINT16(bit(settings_part_t::mqtt), parts());
+  TEST_ASSERT_GOOD(expand(R"({"se":"radio","bl":56})"));
+  TEST_ASSERT_EQUAL_UINT16(bit(settings_part_t::radio), parts());
+  TEST_ASSERT_GOOD(expand(R"({"se":"security","sy":0})"));
+  TEST_ASSERT_EQUAL_UINT16(bit(settings_part_t::security), parts());
+}
+
+int main(int argc, char **argv) {
+  UNITY_BEGIN();
+  RUN_TEST(test_every_section_by_its_name);
+  RUN_TEST(test_a_section_must_be_named);
+  RUN_TEST(test_short_keys_become_the_route_names);
+  RUN_TEST(test_fields_named_apart_from_the_dictionary);
+  RUN_TEST(test_booleans_become_true_and_false);
+  RUN_TEST(test_values_are_checked_against_their_kind);
+  RUN_TEST(test_a_number_may_be_whole);
+  RUN_TEST(test_whole_numbers_are_checked_against_what_the_setting_holds);
+  RUN_TEST(test_the_bit_length_is_56_or_80);
+  RUN_TEST(test_the_radio_proto_is_one_there_is);
+  RUN_TEST(test_numbers_are_checked_against_what_the_radio_can_be_set_to);
+  RUN_TEST(test_strings_must_fit_their_setting);
+  RUN_TEST(test_addresses_are_checked_and_empty_clears_them);
+  RUN_TEST(test_only_what_was_sent_is_in_the_request);
+  RUN_TEST(test_a_save_needs_something_to_save);
+  RUN_TEST(test_an_unknown_key_is_refused);
+  RUN_TEST(test_what_is_only_sent_is_refused);
+  RUN_TEST(test_a_field_of_another_section_is_refused);
+  RUN_TEST(test_secure_config_only_is_not_a_field);
+  RUN_TEST(test_ethernet_needs_a_chip_with_it);
+  RUN_TEST(test_a_save_sets_the_parts_it_has_fields_of);
+  RUN_TEST(test_a_route_name_belongs_to_its_section);
+  return UNITY_END();
+}