feat: web UI protocol 2 client layer, JS and CSS bundles (step 2a)

- data-src/js: ProtoSchema (schema from the sync, short keys and names,
  defaults, cached by hash), ProtoModel (rooms, shades, groups, repeaters,
  header and status from the sync and the events) and ProtoClient (/ws,
  hello, sync with next, op() as a Promise of its reply, events, feeds,
  reconnect). Nothing uses them yet: 2b moves the UI onto them.
- minify.py joins js/*.js and index.js into index.js, and the three CSS
  files into app.css, so a page load opens fewer connections.
- test/js: node --test on the three modules, replaying the recorded
  fixtures; a CI job runs it and the firmware build waits for it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.github/workflows/build.yaml

@@ -28,9 +28,26 @@ jobs:
       - name: Run native unit tests
         run: pio test -e native
 
+  webui:
+    name: Web UI tests
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+
+      - name: Set up Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+
+      # The protocol 2 client (data-src/js) against the recorded fixtures.
+      - name: Run web UI tests
+        run: node --test "test/js/*.test.js"
+
   build:
     # Don't build or publish firmware that fails its tests.
-    needs: test
+    needs: [test, webui]
     permissions: write-all
     name: ${{ matrix.name }}
     runs-on: ubuntu-latest

AGENTS.md

@@ -2,6 +2,7 @@
 - Fix any test or type errors until the whole suite is green.
 - Add or update tests for the code you change, even if nobody asked.
 - to test run `pio test -e native`
+- the web UI's protocol 2 client (`data-src/js/`): `node --test "test/js/*.test.js"`. The minify step joins those files into `index.js`, so each one ends with a `module.exports` footer for Node and adds nothing global but its classes.
 
 ## Building
 - Firmware builds need pioarduino's Core (`pip install pioarduino`, or the pioarduino IDE extension). Stock PlatformIO Core refuses the pinned platform, see [ADR 0003](docs/adr/0003-pioarduino-idf55.md).

data-src/index.html

@@ -8,9 +8,7 @@
     <meta name="apple-mobile-web-app-title" content="ESPSomfy RTS App">
     <meta name="apple-mobile-web-app-status-bar-style" content="black">
 
-    <link rel="stylesheet" href="main.css?v=3.0.14d" type="text/css" />
+    <link rel="stylesheet" href="app.css?v=3.0.14d" type="text/css" />
-    <link rel="stylesheet" href="widgets.css?v=3.0.14d" type="text/css" />
-    <link rel="stylesheet" href="icons.css?v=3.0.14d" type="text/css" />
     <link rel="icon" type="image/png" href="favicon.png" />
 
     <!-- iPad retina icon -->

data-src/js/client.js

@@ -0,0 +1,242 @@
+// A socket protocol 2 client (docs/PROTOCOL.md): ws://host/ws, hello, the sync
+// with next after each frame, ops answered by their reply, events and feeds.
+// It keeps the shade model (ProtoModel) up to date and connects again when the
+// socket closes.  The WebSocket, the timers and the schema's storage can be
+// handed in, so the tests (test/js) run it without a device.  The minify step
+// joins this file into index.js after schema.js and model.js.
+class ProtoError extends Error {
+    // code is an error name from the schema (badRequest, busy ...), or
+    // notConnected, closed or timeout from the client itself.
+    constructor(code, desc, data) {
+        super(desc || code);
+        this.code = code;
+        this.desc = desc || code;
+        this.data = data || {};
+    }
+}
+class ProtoClient {
+    // opts: host, token, WebSocket, storage (getItem/setItem, for the schema),
+    // setTimeout/clearTimeout, opTimeoutMs.
+    constructor(opts) {
+        const deps = ProtoClient.deps();
+        this.Schema = deps.ProtoSchema;
+        this.host = opts.host;
+        this.token = opts.token || '';
+        this.WebSocket = opts.WebSocket || (typeof WebSocket !== 'undefined' ? WebSocket : null);
+        this.storage = opts.storage || null;
+        this.setTimeout = opts.setTimeout || ((fn, ms) => setTimeout(fn, ms));
+        this.clearTimeout = opts.clearTimeout || ((t) => clearTimeout(t));
+        this.opTimeoutMs = opts.opTimeoutMs || 10000;
+        this.schema = this.loadSchema() || new this.Schema();
+        this.model = new deps.ProtoModel(this.schema);
+        this.model.onChange((kind, id, item) => this.emit('change', { kind: kind, id: id, item: item }));
+        this.listeners = {};
+        this.pending = new Map();
+        this.nextId = 1;
+        this.ws = null;
+        this.ready = false;
+        this.opened = false;
+        this.stopped = true;
+        this.retries = 0;
+        this.reconnectTimer = null;
+        this.held = [];
+        this.fw = null;
+    }
+    // In the browser the classes are the ones index.js was joined with; in
+    // Node they are loaded.
+    static deps() {
+        if (typeof require === 'function' && typeof module !== 'undefined' && module.exports)
+            return { ProtoSchema: require('./schema.js').ProtoSchema, ProtoModel: require('./model.js').ProtoModel };
+        return { ProtoSchema: ProtoSchema, ProtoModel: ProtoModel };
+    }
+
+    // cb(d) for a device event by its name (shadeState, frame ...), and for
+    // the client's own: open, hello, synced, change, refused (closed before it
+    // opened: the token may have been refused), closed, versionMismatch and
+    // error (a refusal the device couldn't tie to an op).
+    on(name, cb) { (this.listeners[name] = this.listeners[name] || []).push(cb); }
+    emit(name, d) { for (const cb of this.listeners[name] || []) cb(d); }
+
+    connect() {
+        this.stopped = false;
+        this.open();
+    }
+    close() {
+        this.stopped = true;
+        if (this.reconnectTimer) this.clearTimeout(this.reconnectTimer);
+        this.reconnectTimer = null;
+        if (this.ws) this.ws.close();
+    }
+    // A new token, from a login: the device checks it as the socket opens.
+    setToken(token) {
+        this.token = token || '';
+        if (this.ws) this.ws.close();
+        else if (!this.stopped) this.open();
+    }
+    url() {
+        const params = [];
+        if (this.token) params.push(`apikey=${encodeURIComponent(this.token)}`);
+        if (this.schema.hash) params.push(`schema=${encodeURIComponent(this.schema.hash)}`);
+        return `ws://${this.host}/ws${params.length ? '?' + params.join('&') : ''}`;
+    }
+    open() {
+        this.reconnectTimer = null;
+        this.ready = false;
+        this.opened = false;
+        this.held = [];
+        const ws = new this.WebSocket(this.url());
+        this.ws = ws;
+        ws.onopen = () => {
+            this.opened = true;
+            this.retries = 0;
+            this.emit('open');
+        };
+        ws.onmessage = (evt) => this.receive(evt.data);
+        ws.onclose = () => {
+            if (this.ws !== ws) return;
+            this.ws = null;
+            this.ready = false;
+            for (const [, p] of this.pending) {
+                this.clearTimeout(p.timer);
+                p.reject(new ProtoError('closed', 'The connection closed'));
+            }
+            this.pending.clear();
+            this.emit(this.opened ? 'closed' : 'refused');
+            if (this.stopped) return;
+            // Soon after a close, slower after each failure, at most 10 s.
+            const ms = Math.min(500 * Math.pow(2, this.retries++), 10000);
+            this.reconnectTimer = this.setTimeout(() => this.open(), ms);
+        };
+    }
+    send(type, d, id) {
+        const frame = { v: ProtoClient.VERSION, t: this.schema.typeNumbers[type] };
+        if (typeof frame.t === 'undefined') throw new ProtoError('unknownOp', `The device has no message ${type}`);
+        if (id) frame.id = id;
+        if (d) frame.d = d;
+        this.ws.send(JSON.stringify(frame));
+    }
+
+    // Sends an op, or a control message that wants a reply, by its name with
+    // its fields by name.  Resolves with the reply's fields, rejects with a
+    // ProtoError.
+    op(name, fields) {
+        return new Promise((resolve, reject) => {
+            if (!this.ready || !this.ws) {
+                reject(new ProtoError('notConnected', 'Not connected to the device'));
+                return;
+            }
+            const id = this.nextId++;
+            try {
+                this.send(name, this.schema.pack(fields || {}), id);
+            } catch (err) {
+                reject(err instanceof ProtoError ? err : new ProtoError('badRequest', err.message));
+                return;
+            }
+            const timer = this.setTimeout(() => {
+                this.pending.delete(id);
+                reject(new ProtoError('timeout', `No reply to ${name}`));
+            }, this.opTimeoutMs);
+            this.pending.set(id, { resolve: resolve, reject: reject, timer: timer });
+        });
+    }
+    subscribe(feed) { return this.op('subscribe', { feed: this.schema.feedNumbers[feed] }); }
+    unsubscribe(feed) { return this.op('unsubscribe', { feed: this.schema.feedNumbers[feed] }); }
+    // The shade model and status again, without the schema.
+    resync() {
+        this.model.beginSync();
+        return this.op('sync');
+    }
+
+    receive(text) {
+        let frame;
+        try {
+            frame = JSON.parse(text);
+        } catch (err) {
+            return;
+        }
+        if (frame.v !== ProtoClient.VERSION) {
+            this.emit('versionMismatch', { device: frame.v, client: ProtoClient.VERSION });
+            return;
+        }
+        const d = frame.d || {};
+        // hello, the sync and syncEnd have fixed numbers: they come before the
+        // schema.
+        switch (frame.t) {
+            case 1: this.hello(d); return;
+            case 4: this.syncFrame(d); return;
+            case 5: this.syncEnd(); return;
+            default: break;
+        }
+        if (!this.ready) this.held.push(frame);
+        else this.dispatch(frame);
+    }
+    hello(d) {
+        this.fw = d.fw;
+        // A schema cached for another firmware is no use: the device sends the
+        // one it has, as it didn't get its hash.
+        if (this.schema.hash !== d.sh) {
+            this.schema = new this.Schema();
+            this.model.schema = this.schema;
+        }
+        this.schema.hash = d.sh;
+        this.model.beginSync();
+        this.emit('hello', d);
+    }
+    syncFrame(d) {
+        this.schema.add(d);
+        const device = {};
+        for (const key of Object.keys(d)) if (!['T', 'F', 'E', 'Q'].includes(key)) device[key] = d[key];
+        // The schema's rows come first, so a frame with the device's sections
+        // has the whole of it.
+        if (Object.keys(device).length) this.model.applySync(this.schema.expand(device));
+        this.ws.send(JSON.stringify({ v: ProtoClient.VERSION, t: 3 }));
+    }
+    syncEnd() {
+        this.model.endSync();
+        if (!this.ready) {
+            this.ready = true;
+            this.saveSchema();
+            const held = this.held;
+            this.held = [];
+            for (const frame of held) this.dispatch(frame);
+        }
+        this.emit('synced', this.model);
+    }
+    dispatch(frame) {
+        const d = this.schema.expand(frame.d || {});
+        if (frame.t === 2) {
+            const p = typeof frame.id !== 'undefined' ? this.pending.get(frame.id) : null;
+            if (!p) {
+                if (!d.ok) this.emit('error', new ProtoError(this.schema.errors[d.code] || 'internal', d.desc, d));
+                return;
+            }
+            this.pending.delete(frame.id);
+            this.clearTimeout(p.timer);
+            if (d.ok) p.resolve(d);
+            else p.reject(new ProtoError(this.schema.errors[d.code] || 'internal', d.desc, d));
+            return;
+        }
+        const name = this.schema.types[frame.t];
+        if (!name) return;
+        this.model.applyEvent(name, d);
+        this.emit(name, d);
+    }
+
+    loadSchema() {
+        if (!this.storage) return null;
+        try {
+            return this.Schema.load(JSON.parse(this.storage.getItem(ProtoClient.SCHEMA_KEY)));
+        } catch (err) {
+            return null;
+        }
+    }
+    saveSchema() {
+        if (!this.storage) return;
+        try {
+            this.storage.setItem(ProtoClient.SCHEMA_KEY, JSON.stringify(this.schema.save()));
+        } catch (err) { /* Private windows may have no storage: the device sends the schema each time. */ }
+    }
+}
+ProtoClient.VERSION = 2;
+ProtoClient.SCHEMA_KEY = 'espsomfyProtoSchema';
+if (typeof module !== 'undefined' && module.exports) module.exports = { ProtoClient, ProtoError };

data-src/js/model.js

@@ -0,0 +1,150 @@
+// The shade model as socket protocol 2 keeps a client up to date with it
+// (docs/PROTOCOL.md): rooms, shades, groups and repeaters from the sync, then
+// changed by the device's events.  Items are inserted or updated by id, so an
+// event during the sync does no harm.  It takes frames already expanded to
+// names (ProtoSchema.expand) and fills in what a frame leaves out at its
+// default.  The minify step joins this file into index.js; Node loads it on
+// its own for the tests (test/js).
+class ProtoModel {
+    constructor(schema) {
+        this.schema = schema;
+        this.rooms = new Map();
+        this.shades = new Map();
+        this.groups = new Map();
+        this.repeaters = [];
+        this.header = {};
+        this.status = {};
+        this.listeners = [];
+        this.seen = null;
+    }
+    // cb(kind, id, item): kind is room, shade, group, roomRemoved,
+    // shadeRemoved, groupRemoved, order, repeaters, header or status.
+    onChange(cb) { this.listeners.push(cb); }
+    changed(kind, id, item) { for (const cb of this.listeners) cb(kind, id, item); }
+
+    // A sync: the items it doesn't list are gone once it ends.
+    beginSync() { this.seen = { rooms: new Set(), shades: new Set(), groups: new Set() }; }
+    applySync(d) {
+        if (d.header) {
+            this.header = this.schema.fill(Object.assign({}, d.header), ProtoModel.FIELDS.header);
+            this.changed('header', 0, this.header);
+        }
+        for (const r of d.rooms || []) this.putRoom(r);
+        for (const s of d.shades || []) this.putShade(s);
+        for (const g of d.groups || []) this.putGroup(g);
+        if (d.repeaters) this.putRepeaters(d.repeaters);
+        if (d.status) {
+            this.putStatus(d.status, ProtoModel.FIELDS.memory);
+            this.putStatus(d.status, this.header.connType === 'Ethernet' ? ProtoModel.FIELDS.ethernet : ProtoModel.FIELDS.wifi);
+        }
+    }
+    endSync() {
+        if (!this.seen) return;
+        const lists = [['rooms', 'roomRemoved'], ['shades', 'shadeRemoved'], ['groups', 'groupRemoved']];
+        for (const [list, removed] of lists) {
+            for (const id of [...this[list].keys()]) {
+                if (this.seen[list].has(id)) continue;
+                this[list].delete(id);
+                this.changed(removed, id);
+            }
+        }
+        this.seen = null;
+    }
+    // An event by its type's name; false for one that isn't about the model.
+    applyEvent(type, d) {
+        switch (type) {
+            case 'room': this.putRoom(d); return true;
+            case 'shade': this.putShade(d); return true;
+            case 'group': this.putGroup(d); return true;
+            case 'shadeState': return this.mergeState(this.shades, 'shade', d.shadeId, d, ProtoModel.FIELDS.shadeState);
+            case 'groupState': return this.mergeState(this.groups, 'group', d.groupId, d, ProtoModel.FIELDS.groupState);
+            case 'roomRemoved': return this.remove(this.rooms, type, d.roomId || 0);
+            case 'shadeRemoved': return this.remove(this.shades, type, d.shadeId || 0);
+            case 'groupRemoved': return this.remove(this.groups, type, d.groupId || 0);
+            case 'roomOrder': return this.order(this.rooms, d.ids || []);
+            case 'shadeOrder': return this.order(this.shades, d.ids || []);
+            case 'groupOrder': return this.order(this.groups, d.ids || []);
+            case 'repeaters': this.putRepeaters(d.repeaters || []); return true;
+            case 'memory': this.putStatus(d, ProtoModel.FIELDS.memory); return true;
+            case 'wifi': this.putStatus(d, ProtoModel.FIELDS.wifi); return true;
+            case 'ethernet': this.putStatus(d, ProtoModel.FIELDS.ethernet); return true;
+            default: return false;
+        }
+    }
+
+    // The lists in their sort order.
+    roomList() { return ProtoModel.sorted(this.rooms); }
+    shadeList() { return ProtoModel.sorted(this.shades); }
+    groupList() { return ProtoModel.sorted(this.groups); }
+    static sorted(map) { return [...map.values()].sort((a, b) => a.sortOrder - b.sortOrder); }
+
+    putRoom(r) { this.put(this.rooms, 'room', r.roomId, r, ProtoModel.FIELDS.room, 'rooms'); }
+    putShade(s) {
+        // Its linked remotes leave out their defaults too.
+        const linked = (s.linkedRemotes || []).map((r) => this.schema.fill(Object.assign({}, r), ProtoModel.FIELDS.linkedRemote));
+        this.put(this.shades, 'shade', s.shadeId, Object.assign({}, s, { linkedRemotes: linked }), ProtoModel.FIELDS.shade, 'shades');
+    }
+    putGroup(g) { this.put(this.groups, 'group', g.groupId, g, ProtoModel.FIELDS.group, 'groups'); }
+    // A whole item: what it had before is replaced.
+    put(map, kind, id, item, fields, list) {
+        id = id || 0;
+        const whole = this.schema.fill(Object.assign({}, item), fields);
+        map.set(id, whole);
+        if (this.seen) this.seen[list].add(id);
+        this.changed(kind, id, whole);
+    }
+    // Part of an item: only its state.  A state of an item not synced yet
+    // waits for the item.
+    mergeState(map, kind, id, d, fields) {
+        const item = map.get(id || 0);
+        if (!item) return false;
+        Object.assign(item, this.schema.fill(Object.assign({}, d), fields));
+        this.changed(kind, item[`${kind}Id`], item);
+        return true;
+    }
+    remove(map, kind, id) {
+        if (!map.delete(id)) return false;
+        this.changed(kind, id);
+        return true;
+    }
+    order(map, ids) {
+        ids.forEach((id, i) => {
+            const item = map.get(id);
+            if (item) item.sortOrder = i;
+        });
+        this.changed('order', 0, ids);
+        return true;
+    }
+    putRepeaters(list) {
+        this.repeaters = list.slice();
+        this.changed('repeaters', 0, this.repeaters);
+    }
+    putStatus(d, fields) {
+        const part = {};
+        for (const name of fields) part[name] = typeof d[name] === 'undefined' ? this.schema.defaultOf(name) : d[name];
+        Object.assign(this.status, part);
+        this.changed('status', 0, this.status);
+    }
+}
+// The fields of each kind of item, as the device writes them
+// (src/ProtoSync.cpp, src/outputs/ProtoSocket.cpp): what a frame that leaves
+// one out means by it.
+ProtoModel.FIELDS = {
+    header: ['maxRooms', 'maxShades', 'maxGroups', 'maxGroupedShades', 'maxLinkedRemotes', 'startingAddress', 'bitLength',
+        'proto', 'serverId', 'hostname', 'model', 'chipModel', 'fw', 'connType'],
+    room: ['roomId', 'name', 'sortOrder'],
+    shadeState: ['shadeId', 'position', 'target', 'direction', 'myPos', 'tiltPosition', 'tiltTarget', 'tiltDirection',
+        'myTiltPos', 'flags', 'lastRollingCode'],
+    groupState: ['groupId', 'flags', 'direction'],
+    linkedRemote: ['remoteAddress', 'lastRollingCode'],
+    group: ['groupId', 'roomId', 'name', 'remoteAddress', 'lastRollingCode', 'bitLength', 'proto', 'sunSensor',
+        'flipCommands', 'flags', 'repeats', 'sortOrder', 'linkedShades'],
+    memory: ['freeHeap', 'maxHeap', 'minHeap', 'totalHeap', 'uptime', 'loopRate', 'loopMaxUs'],
+    wifi: ['ssid', 'strength', 'channel'],
+    ethernet: ['connected', 'speed', 'fullDuplex']
+};
+ProtoModel.FIELDS.shade = ProtoModel.FIELDS.shadeState.concat(['roomId', 'name', 'remoteAddress', 'upTime', 'downTime',
+    'tiltTime', 'stepSize', 'paired', 'tiltType', 'shadeType', 'bitLength', 'proto', 'flipCommands', 'flipPosition',
+    'inGroup', 'sunSensor', 'light', 'simMy', 'repeats', 'sortOrder', 'gpioUp', 'gpioDown', 'gpioMy', 'gpioLLTrigger',
+    'linkedRemotes']);
+if (typeof module !== 'undefined' && module.exports) module.exports = { ProtoModel };

data-src/js/schema.js

@@ -0,0 +1,96 @@
+// Socket protocol 2's schema (docs/PROTOCOL.md): the device's table of message
+// types, fields, error codes and feeds, as the sync sends it.  It turns the
+// short keys of a frame into names and back.  The minify step joins this file
+// into index.js; Node loads it on its own for the tests (test/js).
+class ProtoSchema {
+    constructor() {
+        this.hash = null;
+        this.types = {};        // number -> name
+        this.typeNumbers = {};  // name -> number
+        this.fields = {};       // key -> { name, kind, def }
+        this.keys = {};         // name -> key
+        this.errors = {};
+        this.errorNumbers = {};
+        this.feeds = {};
+        this.feedNumbers = {};
+        this.rows = { T: [], F: [], E: [], Q: [] };
+    }
+    // The schema's rows in a sync frame's payload: T types, F fields, E error
+    // codes, Q feeds.  They may span several frames.
+    add(d) {
+        for (const [number, name] of d.T || []) {
+            this.types[number] = name;
+            this.typeNumbers[name] = number;
+        }
+        for (const [key, name, kind, def] of d.F || []) {
+            this.fields[key] = { name: name, kind: kind, def: def };
+            this.keys[name] = key;
+        }
+        for (const [number, name] of d.E || []) {
+            this.errors[number] = name;
+            this.errorNumbers[name] = number;
+        }
+        for (const [number, name] of d.Q || []) {
+            this.feeds[number] = name;
+            this.feedNumbers[name] = number;
+        }
+        for (const k of ['T', 'F', 'E', 'Q']) if (d[k]) this.rows[k].push(...d[k]);
+    }
+    get empty() { return Object.keys(this.fields).length === 0; }
+    // What a field left out of a frame is: its default.
+    defaultOf(name) {
+        const f = this.fields[this.keys[name]];
+        if (!f) return undefined;
+        switch (f.kind) {
+            case 'b': return f.def !== 0;
+            case 's': return '';
+            case 'l': return [];
+            case 'o': return {};
+            default: return f.def;
+        }
+    }
+    // Short keys to names, and 1/0 to true/false, all the way down.  A field
+    // left out stays out; fill() puts the defaults in.
+    expand(value) {
+        if (Array.isArray(value)) return value.map((v) => this.expand(v));
+        if (value === null || typeof value !== 'object') return value;
+        const out = {};
+        for (const key of Object.keys(value)) {
+            const f = this.fields[key];
+            let v = value[key];
+            if (f && f.kind === 'b' && typeof v === 'number') v = v !== 0;
+            out[f ? f.name : key] = this.expand(v);
+        }
+        return out;
+    }
+    // Names to short keys, and booleans to 1/0, for an op's payload.  A name
+    // the schema doesn't have is an error: the device would refuse it anyway.
+    pack(value) {
+        if (Array.isArray(value)) return value.map((v) => this.pack(v));
+        if (value === null || typeof value !== 'object') return typeof value === 'boolean' ? (value ? 1 : 0) : value;
+        const out = {};
+        for (const name of Object.keys(value)) {
+            if (typeof value[name] === 'undefined') continue;
+            const key = this.keys[name];
+            if (typeof key === 'undefined') throw new Error(`The device has no field ${name}`);
+            out[key] = this.pack(value[name]);
+        }
+        return out;
+    }
+    // Sets each of names that obj lacks to its default, as a frame leaves out
+    // a field at its default.  Returns obj.
+    fill(obj, names) {
+        for (const name of names) if (typeof obj[name] === 'undefined') obj[name] = this.defaultOf(name);
+        return obj;
+    }
+    // What a cache keeps, and the schema back from it.
+    save() { return { hash: this.hash, rows: this.rows }; }
+    static load(saved) {
+        if (!saved || typeof saved.hash !== 'string' || !saved.rows) return null;
+        const s = new ProtoSchema();
+        s.add(saved.rows);
+        s.hash = saved.hash;
+        return s.empty ? null : s;
+    }
+}
+if (typeof module !== 'undefined' && module.exports) module.exports = { ProtoSchema };

data-src/login.html

@@ -3,9 +3,7 @@
 <head>
     <meta name="viewport" content="width=device-width, initial-scale=1">
     <meta charset="UTF-8">
-    <link rel="stylesheet" href="main.css?v=2.4.0" type="text/css" />
+    <link rel="stylesheet" href="app.css?v=2.4.0" type="text/css" />
-    <link rel="stylesheet" href="widgets.css?v=2.4.0" type="text/css" />
-    <link rel="stylesheet" href="icons.css?v=2.4.0" type="text/css" />
     <link rel="icon" type="image/png" href="favicon.png" />
     <script type="text/javascript" src="index.js?v=2.4.0"></script>
 </head>

minify.py

@@ -52,6 +52,16 @@ MINIFY_AND_GZIP = {".html", ".htm", ".css", ".js", ".json", ".svg", ".xml"}
 # Extensions that optipng can optimise
 PNG_EXTENSIONS = {".png"}
 
+# Files joined into one, in this order, before they are minified: a page load
+# then opens fewer connections, each of which costs the device heap
+# (docs/adr/0005-one-websocket-protocol.md).  The protocol 2 client in js/
+# comes before index.js, which uses it.  A source listed here isn't copied on
+# its own.
+BUNDLES = {
+    "index.js": ["js/schema.js", "js/model.js", "js/client.js", "index.js"],
+    "app.css": ["main.css", "widgets.css", "icons.css"],
+}
+
 # Everything else is copied verbatim
 # ──────────────────────────────────────────────
 
@@ -252,6 +262,33 @@ def minify_all():
     total_final = 0
     file_count = 0
 
+    def report(rel_path, stats):
+        nonlocal total_original, total_final, file_count
+        pct = (1 - stats["final"] / stats["original"]) * 100 if stats["original"] > 0 else 0
+        print(
+            f"  {rel_path:<40s} "
+            f"{stats['original']:>8,d} → {stats['final']:>8,d} B "
+            f"({pct:+.0f}%)  [{stats['action']}]"
+        )
+        total_original += stats["original"]
+        total_final += stats["final"]
+        file_count += 1
+
+    bundled = set()
+    work_dir = os.path.join(dst_dir, ".bundles")
+    os.makedirs(work_dir, exist_ok=True)
+    for name, parts in BUNDLES.items():
+        joined = os.path.join(work_dir, name)
+        with open(joined, "w", encoding="utf-8") as out:
+            for part in parts:
+                with open(os.path.join(src_dir, part), "r", encoding="utf-8") as f:
+                    out.write(f.read())
+                # A file without a final newline would run into the next.
+                out.write("\n")
+                bundled.add(os.path.normpath(part))
+        report(f"{name} ({len(parts)} files)", process_file(joined, os.path.join(dst_dir, name)))
+    shutil.rmtree(work_dir)
+
     for root, dirs, files in os.walk(src_dir):
         for fname in sorted(files):
             # Skip hidden files and editor temp files
@@ -260,20 +297,11 @@ def minify_all():
 
             src_path = os.path.join(root, fname)
             rel_path = os.path.relpath(src_path, src_dir)
+            if os.path.normpath(rel_path) in bundled:
+                continue
             dst_path = os.path.join(dst_dir, rel_path)
 
-            stats = process_file(src_path, dst_path)
+            report(rel_path, process_file(src_path, dst_path))
-
-            pct = (1 - stats["final"] / stats["original"]) * 100 if stats["original"] > 0 else 0
-            print(
-                f"  {rel_path:<40s} "
-                f"{stats['original']:>8,d} → {stats['final']:>8,d} B "
-                f"({pct:+.0f}%)  [{stats['action']}]"
-            )
-
-            total_original += stats["original"]
-            total_final += stats["final"]
-            file_count += 1
 
     print(f"[minify] {file_count} files processed")
     print(

test/e2e/test_survival.py

@@ -54,7 +54,7 @@ def test_one_client_too_many_is_closed(connect, schema):
 
 def test_page_load(host, token, connect, watch_heap):
     """The static files in parallel, as a browser fetches them, then the socket."""
-    files = ["/", "/main.css", "/widgets.css", "/icons.css", "/index.js", "/icon.png"]
+    files = ["/", "/app.css", "/index.js", "/icon.png"]
     errors = []
 
     def get(path):

test/js/client.test.js

@@ -0,0 +1,200 @@
+// The web UI's ProtoClient (data-src/js/client.js): the recorded hello and
+// sync replayed through a stand-in socket.
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const { ProtoClient, ProtoError } = require('../../data-src/js/client.js');
+const { hello, sync, recordedItems, recordedSchema, FakeSocket, FakeTimers, FakeStorage } = require('./helpers.js');
+
+function client(opts) {
+    FakeSocket.reset();
+    const timers = new FakeTimers();
+    const c = new ProtoClient(Object.assign({ host: 'device', WebSocket: FakeSocket, setTimeout: timers.setTimeout,
+        clearTimeout: timers.clearTimeout }, opts || {}));
+    c.timers = timers;
+    return c;
+}
+// Connects and plays the recorded hello and sync.
+function connected(opts) {
+    const c = client(opts);
+    c.connect();
+    const ws = FakeSocket.last;
+    ws.open();
+    ws.deliver(hello);
+    for (const f of sync) ws.deliver(f);
+    return { c, ws };
+}
+const schema = recordedSchema();
+const T = schema.typeNumbers;
+function reply(id, d) { return { v: 2, t: T.reply, id: id, d: d }; }
+
+test('the client connects with its token and syncs, sending next after each sync frame', () => {
+    const c = client({ token: 'abc 123' });
+    c.connect();
+    const ws = FakeSocket.last;
+    assert.equal(ws.url, 'ws://device/ws?apikey=abc%20123');
+    ws.open();
+    ws.deliver(hello);
+    for (const f of sync) ws.deliver(f);
+    const syncFrames = sync.filter((f) => f.t === T.sync).length;
+    assert.equal(ws.sent.filter((f) => f.t === T.next).length, syncFrames);
+    assert.ok(c.ready);
+    assert.equal(c.fw, hello.d.fw);
+    assert.equal(c.model.shades.size, recordedItems('S').length);
+});
+
+test('synced comes once the sync has ended', () => {
+    const c = client();
+    let synced = 0;
+    c.on('synced', () => synced++);
+    c.connect();
+    const ws = FakeSocket.last;
+    ws.open();
+    ws.deliver(hello);
+    for (const f of sync.filter((f) => f.t === T.sync)) ws.deliver(f);
+    assert.equal(synced, 0);
+    ws.deliver(sync.find((f) => f.t === T.syncEnd));
+    assert.equal(synced, 1);
+});
+
+test('an op resolves with its reply', async () => {
+    const { c, ws } = connected();
+    const p = c.op('getSettings', { section: 'radio' });
+    const sent = ws.sent[ws.sent.length - 1];
+    assert.deepEqual(sent, { v: 2, t: T.getSettings, id: sent.id, d: { se: 'radio' } });
+    ws.deliver(reply(sent.id, { k: 1, bl: 80, en: 1 }));
+    assert.deepEqual(await p, { ok: true, bitLength: 80, enabled: true });
+});
+
+test('a refused op rejects with the error by its name', async () => {
+    const { c, ws } = connected();
+    const p = c.op('saveSettings', { section: 'mqtt' });
+    const id = ws.sent[ws.sent.length - 1].id;
+    ws.deliver(reply(id, { e: schema.errorNumbers.badRequest, x: 'Nothing to save' }));
+    await assert.rejects(p, (err) => err instanceof ProtoError && err.code === 'badRequest' && err.desc === 'Nothing to save');
+});
+
+test('an op is refused before the sync, and when its reply never comes', async () => {
+    const c = client();
+    await assert.rejects(c.op('reboot'), (err) => err.code === 'notConnected');
+    const { c: c2 } = connected();
+    const p = c2.op('reboot');
+    c2.timers.runAll();
+    await assert.rejects(p, (err) => err.code === 'timeout');
+});
+
+test('an op of a name or field the device lacks is refused here', async () => {
+    const { c } = connected();
+    await assert.rejects(c.op('noSuchOp'), (err) => err.code === 'unknownOp');
+    await assert.rejects(c.op('reboot', { noSuchField: 1 }), (err) => err.code === 'badRequest');
+});
+
+test('ops waiting when the socket closes are refused, and the client connects again', async () => {
+    const { c, ws } = connected();
+    const p = c.op('scanWifi');
+    let closed = 0;
+    c.on('closed', () => closed++);
+    ws.drop();
+    await assert.rejects(p, (err) => err.code === 'closed');
+    assert.equal(closed, 1);
+    assert.equal(c.ready, false);
+    assert.deepEqual(c.timers.runAll(), [500]);
+    assert.notEqual(FakeSocket.last, ws);
+});
+
+test('a socket refused before it opened says so, and the retries slow down', () => {
+    const c = client();
+    let refused = 0;
+    c.on('refused', () => refused++);
+    c.connect();
+    const delays = [];
+    for (let i = 0; i < 6; i++) {
+        FakeSocket.last.drop();
+        delays.push(...c.timers.runAll());
+    }
+    assert.equal(refused, 6);
+    assert.deepEqual(delays, [500, 1000, 2000, 4000, 8000, 10000]);
+});
+
+test('close stops the reconnecting', () => {
+    const { c } = connected();
+    c.close();
+    assert.deepEqual(c.timers.runAll(), []);
+});
+
+test('events update the model, and are told by name', () => {
+    const { c, ws } = connected();
+    const shade = c.model.shadeList()[0];
+    const seen = [];
+    c.on('shadeState', (d) => seen.push(d));
+    ws.deliver({ v: 2, t: T.shadeState, d: { s: shade.shadeId, t: 30, d: 1 } });
+    assert.equal(c.model.shades.get(shade.shadeId).target, 30);
+    assert.deepEqual(seen, [{ shadeId: shade.shadeId, target: 30, direction: 1 }]);
+});
+
+test('an event before the schema is kept until the sync ends', () => {
+    const c = client();
+    c.connect();
+    const ws = FakeSocket.last;
+    ws.open();
+    ws.deliver(hello);
+    // Before the schema: the client can't read it yet.
+    const shadeId = recordedItems('S')[0].s;
+    ws.deliver({ v: 2, t: T.shadeState, d: { s: shadeId, t: 55 } });
+    for (const f of sync) ws.deliver(f);
+    assert.equal(c.model.shades.get(shadeId).target, 55);
+});
+
+test('the schema is kept, and the device is told it has it', () => {
+    const storage = new FakeStorage();
+    connected({ storage: storage });
+    const c = client({ storage: storage });
+    c.connect();
+    const ws = FakeSocket.last;
+    assert.equal(ws.url, `ws://device/ws?schema=${hello.d.sh}`);
+    ws.open();
+    ws.deliver(hello);
+    // Without the schema's rows, as the device sends a client that has it.
+    for (const f of sync) {
+        const d = Object.assign({}, f.d);
+        for (const k of ['T', 'F', 'E', 'Q']) delete d[k];
+        ws.deliver(Object.assign({}, f, { d: d }));
+    }
+    assert.ok(c.ready);
+    assert.equal(c.model.shades.size, recordedItems('S').length);
+});
+
+test('a kept schema of another firmware is dropped', () => {
+    const storage = new FakeStorage();
+    connected({ storage: storage });
+    const c = client({ storage: storage });
+    c.connect();
+    const ws = FakeSocket.last;
+    ws.open();
+    ws.deliver({ v: 2, t: 1, d: { fw: 'v4.0.0', sh: 'ffffffff' } });
+    assert.equal(c.schema.hash, 'ffffffff');
+    assert.ok(c.schema.empty);
+});
+
+test('a frame of another protocol version is reported', () => {
+    const { c, ws } = connected();
+    let mismatch = null;
+    c.on('versionMismatch', (d) => { mismatch = d; });
+    ws.deliver({ v: 3, t: 1, d: {} });
+    assert.deepEqual(mismatch, { device: 3, client: 2 });
+});
+
+test('a new token connects again with it', () => {
+    const { c, ws } = connected();
+    c.setToken('new');
+    assert.ok(ws.closed);
+    c.timers.runAll();
+    assert.match(FakeSocket.last.url, /apikey=new/);
+});
+
+test('subscribe sends the feed number', () => {
+    const { c, ws } = connected();
+    c.subscribe('frames');
+    const sent = ws.sent[ws.sent.length - 1];
+    assert.equal(sent.t, T.subscribe);
+    assert.deepEqual(sent.d, { fe: schema.feedNumbers.frames });
+});

test/js/helpers.js

@@ -0,0 +1,82 @@
+// What the web UI client tests share: the frames the e2e suite recorded from a
+// device (test/fixtures/proto2), and stand-ins for the socket and the timers.
+const path = require('node:path');
+
+const FIXTURES = path.join(__dirname, '..', 'fixtures', 'proto2');
+const hello = require(path.join(FIXTURES, 'hello.json'));
+const sync = require(path.join(FIXTURES, 'sync.json'));
+const schemaJson = require(path.join(FIXTURES, 'schema.json'));
+const { ProtoSchema } = require('../../data-src/js/schema.js');
+
+// A schema filled from the recorded sync, as a client builds it.
+function recordedSchema() {
+    const s = new ProtoSchema();
+    for (const f of sync) if (f.t === 4) s.add(f.d);
+    s.hash = hello.d.sh;
+    return s;
+}
+// The recorded sync's device sections, expanded, frame by frame.
+function deviceSections(schema) {
+    return sync.filter((f) => f.t === 4).map((f) => {
+        const d = {};
+        for (const k of Object.keys(f.d)) if (!['T', 'F', 'E', 'Q'].includes(k)) d[k] = f.d[k];
+        return schema.expand(d);
+    });
+}
+// The recorded sync's items under their short keys.
+function recordedItems(key) {
+    return sync.filter((f) => f.t === 4 && f.d[key]).flatMap((f) => f.d[key]);
+}
+
+// A WebSocket the test drives: it records what the client sends, and the test
+// opens, delivers and closes.
+class FakeSocket {
+    constructor(url) {
+        this.url = url;
+        this.sent = [];
+        this.closed = false;
+        FakeSocket.last = this;
+        FakeSocket.all.push(this);
+    }
+    send(text) { this.sent.push(JSON.parse(text)); }
+    close() {
+        if (this.closed) return;
+        this.closed = true;
+        if (this.onclose) this.onclose({});
+    }
+    open() { if (this.onopen) this.onopen({}); }
+    deliver(frame) { this.onmessage({ data: typeof frame === 'string' ? frame : JSON.stringify(frame) }); }
+    // The device closing it, or refusing it before it opened.
+    drop() { this.close(); }
+}
+FakeSocket.all = [];
+FakeSocket.reset = () => { FakeSocket.all = []; FakeSocket.last = null; };
+
+// Timers the test runs by hand.
+class FakeTimers {
+    constructor() {
+        this.timers = new Map();
+        this.next = 1;
+        this.setTimeout = (fn, ms) => {
+            const id = this.next++;
+            this.timers.set(id, { fn: fn, ms: ms });
+            return id;
+        };
+        this.clearTimeout = (id) => { this.timers.delete(id); };
+    }
+    // Runs every timer waiting, and returns their delays.
+    runAll() {
+        const due = [...this.timers.entries()];
+        this.timers.clear();
+        for (const [, t] of due) t.fn();
+        return due.map(([, t]) => t.ms);
+    }
+}
+// localStorage as far as the client uses it.
+class FakeStorage {
+    constructor() { this.items = {}; }
+    getItem(key) { return Object.prototype.hasOwnProperty.call(this.items, key) ? this.items[key] : null; }
+    setItem(key, value) { this.items[key] = String(value); }
+}
+
+module.exports = { hello, sync, schemaJson, recordedSchema, deviceSections, recordedItems, FakeSocket, FakeTimers, FakeStorage };

test/js/model.test.js

@@ -0,0 +1,106 @@
+// The web UI's ProtoModel (data-src/js/model.js): the recorded sync, then
+// events.
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const { ProtoModel } = require('../../data-src/js/model.js');
+const { recordedSchema, deviceSections, recordedItems } = require('./helpers.js');
+
+function synced() {
+    const schema = recordedSchema();
+    const model = new ProtoModel(schema);
+    model.beginSync();
+    for (const d of deviceSections(schema)) model.applySync(d);
+    model.endSync();
+    return model;
+}
+
+test('the sync fills every room, shade and group', () => {
+    const m = synced();
+    assert.equal(m.rooms.size, recordedItems('R').length);
+    assert.equal(m.shades.size, recordedItems('S').length);
+    assert.equal(m.groups.size, recordedItems('G').length);
+    assert.equal(m.header.model, 'ESPSomfyRTS');
+    assert.equal(m.header.maxShades, 32);
+    assert.ok(m.status.freeHeap > 0);
+});
+
+test('a field the sync left out is its default', () => {
+    const m = synced();
+    const raw = recordedItems('S').find((s) => typeof s.m === 'undefined');
+    const shade = m.shades.get(raw.s);
+    // Left out because they were at their default.
+    assert.equal(shade.myPos, -1);
+    assert.equal(shade.stepSize, raw.z === undefined ? 100 : raw.z);
+    assert.equal(typeof shade.paired, 'boolean');
+    assert.ok(Array.isArray(shade.linkedRemotes));
+    for (const r of shade.linkedRemotes) assert.equal(typeof r.lastRollingCode, 'number');
+    for (const g of m.groupList()) assert.ok(Array.isArray(g.linkedShades));
+});
+
+test('the lists come in their sort order', () => {
+    const m = synced();
+    const orders = m.shadeList().map((s) => s.sortOrder);
+    assert.deepEqual(orders, [...orders].sort((a, b) => a - b));
+});
+
+test('a state event changes only the state, and a left-out field is its default', () => {
+    const m = synced();
+    const shade = m.shadeList()[0];
+    const name = shade.name;
+    // A shade at rest at the top: position 0 is left out.
+    assert.ok(m.applyEvent('shadeState', { shadeId: shade.shadeId, target: 40, direction: 1 }));
+    assert.equal(m.shades.get(shade.shadeId).position, 0);
+    assert.equal(m.shades.get(shade.shadeId).target, 40);
+    assert.equal(m.shades.get(shade.shadeId).name, name);
+    // Of a shade the client doesn't have yet: nothing to change.
+    assert.equal(m.applyEvent('shadeState', { shadeId: 250, position: 5 }), false);
+});
+
+test('a whole item replaces what was there', () => {
+    const m = synced();
+    const shade = m.shadeList()[0];
+    m.applyEvent('shade', { shadeId: shade.shadeId, name: 'Renamed' });
+    assert.equal(m.shades.get(shade.shadeId).name, 'Renamed');
+    assert.equal(m.shades.get(shade.shadeId).upTime, 0);
+    m.applyEvent('room', { roomId: 9, name: 'New' });
+    assert.equal(m.rooms.get(9).sortOrder, 0);
+});
+
+test('removals, orders and repeaters', () => {
+    const m = synced();
+    const changes = [];
+    m.onChange((kind, id) => changes.push([kind, id]));
+    const [first, second] = m.shadeList();
+    assert.ok(m.applyEvent('shadeRemoved', { shadeId: first.shadeId }));
+    assert.equal(m.shades.has(first.shadeId), false);
+    // An id it doesn't have is ignored.
+    assert.equal(m.applyEvent('roomRemoved', { roomId: 250 }), false);
+    m.applyEvent('shadeOrder', { ids: [second.shadeId] });
+    assert.equal(m.shades.get(second.shadeId).sortOrder, 0);
+    m.applyEvent('repeaters', { repeaters: [7, 8] });
+    assert.deepEqual(m.repeaters, [7, 8]);
+    assert.deepEqual(changes.map((c) => c[0]), ['shadeRemoved', 'order', 'repeaters']);
+});
+
+test('a sync drops what it no longer lists, and keeps what an event added during it', () => {
+    const m = synced();
+    const schema = m.schema;
+    m.applyEvent('room', { roomId: 99, name: 'Gone' });
+    m.beginSync();
+    m.applyEvent('room', { roomId: 98, name: 'Added during the sync' });
+    for (const d of deviceSections(schema)) m.applySync(d);
+    m.endSync();
+    assert.equal(m.rooms.has(99), false);
+    assert.equal(m.rooms.has(98), true);
+});
+
+test('status events', () => {
+    const m = synced();
+    m.applyEvent('memory', { freeHeap: 1000, uptime: 5 });
+    assert.equal(m.status.freeHeap, 1000);
+    assert.equal(m.status.loopRate, 0);
+    m.applyEvent('wifi', { strength: -60, channel: 6 });
+    assert.equal(m.status.ssid, '');
+    assert.equal(m.status.strength, -60);
+    assert.equal(m.applyEvent('frame', { remoteAddress: 1 }), false);
+});

test/js/schema.test.js

@@ -0,0 +1,56 @@
+// The web UI's ProtoSchema (data-src/js/schema.js) against the schema a
+// device sent (test/fixtures/proto2).
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const { ProtoSchema } = require('../../data-src/js/schema.js');
+const { recordedSchema, schemaJson } = require('./helpers.js');
+
+test('the schema is collected from every sync frame it spans', () => {
+    const s = recordedSchema();
+    assert.equal(Object.keys(s.types).length, Object.keys(schemaJson.types).length);
+    assert.equal(Object.keys(s.fields).length, Object.keys(schemaJson.fields).length);
+    assert.equal(s.typeNumbers.shadeCommand, 16);
+    assert.equal(s.typeNumbers.saveSettings, 50);
+    assert.equal(s.keys.shadeId, 's');
+    assert.equal(s.errorNumbers.busy, 5);
+    assert.equal(s.feedNumbers.frequencyScan, 2);
+});
+
+test('short keys become names, and 1/0 booleans, all the way down', () => {
+    const s = recordedSchema();
+    const shade = s.expand({ s: 3, n: 'Office', i: 1, fp: 0, lk: [{ a: 5, c: 9 }] });
+    assert.deepEqual(shade, { shadeId: 3, name: 'Office', paired: true, flipPosition: false,
+        linkedRemotes: [{ remoteAddress: 5, lastRollingCode: 9 }] });
+    // A whole number that happens to be 1 stays a number.
+    assert.deepEqual(s.expand({ p: 1 }), { position: 1 });
+});
+
+test('names become short keys for an op', () => {
+    const s = recordedSchema();
+    assert.deepEqual(s.pack({ shadeId: 3, command: 'Up', paired: true, sunny: false }), { s: 3, cm: 'Up', i: 1, sn: 0 });
+    assert.deepEqual(s.pack({ ids: [3, 1, 2] }), { is: [3, 1, 2] });
+    // Left out, not sent as undefined.
+    assert.deepEqual(s.pack({ shadeId: 3, target: undefined }), { s: 3 });
+    assert.throws(() => s.pack({ nonsense: 1 }), /no field nonsense/);
+});
+
+test('a field left out is its default', () => {
+    const s = recordedSchema();
+    assert.equal(s.defaultOf('myPos'), -1);
+    assert.equal(s.defaultOf('stepSize'), 100);
+    assert.equal(s.defaultOf('bitLength'), 56);
+    assert.equal(s.defaultOf('paired'), false);
+    assert.equal(s.defaultOf('name'), '');
+    assert.deepEqual(s.defaultOf('linkedShades'), []);
+    assert.deepEqual(s.fill({ shadeId: 2 }, ['shadeId', 'position', 'myPos']), { shadeId: 2, position: 0, myPos: -1 });
+});
+
+test('a cached schema comes back whole, with its hash', () => {
+    const s = recordedSchema();
+    const back = ProtoSchema.load(JSON.parse(JSON.stringify(s.save())));
+    assert.equal(back.hash, s.hash);
+    assert.deepEqual(back.fields, s.fields);
+    assert.deepEqual(back.types, s.types);
+    assert.equal(ProtoSchema.load(null), null);
+    assert.equal(ProtoSchema.load({ hash: 'x', rows: {} }), null);
+});