feat: web UI protocol 2 client layer, JS and CSS bundles (step 2a)

- data-src/js: ProtoSchema (schema from the sync, short keys and names,
  defaults, cached by hash), ProtoModel (rooms, shades, groups, repeaters,
  header and status from the sync and the events) and ProtoClient (/ws,
  hello, sync with next, op() as a Promise of its reply, events, feeds,
  reconnect). Nothing uses them yet: 2b moves the UI onto them.
- minify.py joins js/*.js and index.js into index.js, and the three CSS
  files into app.css, so a page load opens fewer connections.
- test/js: node --test on the three modules, replaying the recorded
  fixtures; a CI job runs it and the firmware build waits for it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

test/e2e/test_survival.py

@@ -54,7 +54,7 @@ def test_one_client_too_many_is_closed(connect, schema):
 
 def test_page_load(host, token, connect, watch_heap):
     """The static files in parallel, as a browser fetches them, then the socket."""
-    files = ["/", "/main.css", "/widgets.css", "/icons.css", "/index.js", "/icon.png"]
+    files = ["/", "/app.css", "/index.js", "/icon.png"]
     errors = []
 
     def get(path):

test/js/client.test.js

@@ -0,0 +1,200 @@
+// The web UI's ProtoClient (data-src/js/client.js): the recorded hello and
+// sync replayed through a stand-in socket.
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const { ProtoClient, ProtoError } = require('../../data-src/js/client.js');
+const { hello, sync, recordedItems, recordedSchema, FakeSocket, FakeTimers, FakeStorage } = require('./helpers.js');
+
+function client(opts) {
+    FakeSocket.reset();
+    const timers = new FakeTimers();
+    const c = new ProtoClient(Object.assign({ host: 'device', WebSocket: FakeSocket, setTimeout: timers.setTimeout,
+        clearTimeout: timers.clearTimeout }, opts || {}));
+    c.timers = timers;
+    return c;
+}
+// Connects and plays the recorded hello and sync.
+function connected(opts) {
+    const c = client(opts);
+    c.connect();
+    const ws = FakeSocket.last;
+    ws.open();
+    ws.deliver(hello);
+    for (const f of sync) ws.deliver(f);
+    return { c, ws };
+}
+const schema = recordedSchema();
+const T = schema.typeNumbers;
+function reply(id, d) { return { v: 2, t: T.reply, id: id, d: d }; }
+
+test('the client connects with its token and syncs, sending next after each sync frame', () => {
+    const c = client({ token: 'abc 123' });
+    c.connect();
+    const ws = FakeSocket.last;
+    assert.equal(ws.url, 'ws://device/ws?apikey=abc%20123');
+    ws.open();
+    ws.deliver(hello);
+    for (const f of sync) ws.deliver(f);
+    const syncFrames = sync.filter((f) => f.t === T.sync).length;
+    assert.equal(ws.sent.filter((f) => f.t === T.next).length, syncFrames);
+    assert.ok(c.ready);
+    assert.equal(c.fw, hello.d.fw);
+    assert.equal(c.model.shades.size, recordedItems('S').length);
+});
+
+test('synced comes once the sync has ended', () => {
+    const c = client();
+    let synced = 0;
+    c.on('synced', () => synced++);
+    c.connect();
+    const ws = FakeSocket.last;
+    ws.open();
+    ws.deliver(hello);
+    for (const f of sync.filter((f) => f.t === T.sync)) ws.deliver(f);
+    assert.equal(synced, 0);
+    ws.deliver(sync.find((f) => f.t === T.syncEnd));
+    assert.equal(synced, 1);
+});
+
+test('an op resolves with its reply', async () => {
+    const { c, ws } = connected();
+    const p = c.op('getSettings', { section: 'radio' });
+    const sent = ws.sent[ws.sent.length - 1];
+    assert.deepEqual(sent, { v: 2, t: T.getSettings, id: sent.id, d: { se: 'radio' } });
+    ws.deliver(reply(sent.id, { k: 1, bl: 80, en: 1 }));
+    assert.deepEqual(await p, { ok: true, bitLength: 80, enabled: true });
+});
+
+test('a refused op rejects with the error by its name', async () => {
+    const { c, ws } = connected();
+    const p = c.op('saveSettings', { section: 'mqtt' });
+    const id = ws.sent[ws.sent.length - 1].id;
+    ws.deliver(reply(id, { e: schema.errorNumbers.badRequest, x: 'Nothing to save' }));
+    await assert.rejects(p, (err) => err instanceof ProtoError && err.code === 'badRequest' && err.desc === 'Nothing to save');
+});
+
+test('an op is refused before the sync, and when its reply never comes', async () => {
+    const c = client();
+    await assert.rejects(c.op('reboot'), (err) => err.code === 'notConnected');
+    const { c: c2 } = connected();
+    const p = c2.op('reboot');
+    c2.timers.runAll();
+    await assert.rejects(p, (err) => err.code === 'timeout');
+});
+
+test('an op of a name or field the device lacks is refused here', async () => {
+    const { c } = connected();
+    await assert.rejects(c.op('noSuchOp'), (err) => err.code === 'unknownOp');
+    await assert.rejects(c.op('reboot', { noSuchField: 1 }), (err) => err.code === 'badRequest');
+});
+
+test('ops waiting when the socket closes are refused, and the client connects again', async () => {
+    const { c, ws } = connected();
+    const p = c.op('scanWifi');
+    let closed = 0;
+    c.on('closed', () => closed++);
+    ws.drop();
+    await assert.rejects(p, (err) => err.code === 'closed');
+    assert.equal(closed, 1);
+    assert.equal(c.ready, false);
+    assert.deepEqual(c.timers.runAll(), [500]);
+    assert.notEqual(FakeSocket.last, ws);
+});
+
+test('a socket refused before it opened says so, and the retries slow down', () => {
+    const c = client();
+    let refused = 0;
+    c.on('refused', () => refused++);
+    c.connect();
+    const delays = [];
+    for (let i = 0; i < 6; i++) {
+        FakeSocket.last.drop();
+        delays.push(...c.timers.runAll());
+    }
+    assert.equal(refused, 6);
+    assert.deepEqual(delays, [500, 1000, 2000, 4000, 8000, 10000]);
+});
+
+test('close stops the reconnecting', () => {
+    const { c } = connected();
+    c.close();
+    assert.deepEqual(c.timers.runAll(), []);
+});
+
+test('events update the model, and are told by name', () => {
+    const { c, ws } = connected();
+    const shade = c.model.shadeList()[0];
+    const seen = [];
+    c.on('shadeState', (d) => seen.push(d));
+    ws.deliver({ v: 2, t: T.shadeState, d: { s: shade.shadeId, t: 30, d: 1 } });
+    assert.equal(c.model.shades.get(shade.shadeId).target, 30);
+    assert.deepEqual(seen, [{ shadeId: shade.shadeId, target: 30, direction: 1 }]);
+});
+
+test('an event before the schema is kept until the sync ends', () => {
+    const c = client();
+    c.connect();
+    const ws = FakeSocket.last;
+    ws.open();
+    ws.deliver(hello);
+    // Before the schema: the client can't read it yet.
+    const shadeId = recordedItems('S')[0].s;
+    ws.deliver({ v: 2, t: T.shadeState, d: { s: shadeId, t: 55 } });
+    for (const f of sync) ws.deliver(f);
+    assert.equal(c.model.shades.get(shadeId).target, 55);
+});
+
+test('the schema is kept, and the device is told it has it', () => {
+    const storage = new FakeStorage();
+    connected({ storage: storage });
+    const c = client({ storage: storage });
+    c.connect();
+    const ws = FakeSocket.last;
+    assert.equal(ws.url, `ws://device/ws?schema=${hello.d.sh}`);
+    ws.open();
+    ws.deliver(hello);
+    // Without the schema's rows, as the device sends a client that has it.
+    for (const f of sync) {
+        const d = Object.assign({}, f.d);
+        for (const k of ['T', 'F', 'E', 'Q']) delete d[k];
+        ws.deliver(Object.assign({}, f, { d: d }));
+    }
+    assert.ok(c.ready);
+    assert.equal(c.model.shades.size, recordedItems('S').length);
+});
+
+test('a kept schema of another firmware is dropped', () => {
+    const storage = new FakeStorage();
+    connected({ storage: storage });
+    const c = client({ storage: storage });
+    c.connect();
+    const ws = FakeSocket.last;
+    ws.open();
+    ws.deliver({ v: 2, t: 1, d: { fw: 'v4.0.0', sh: 'ffffffff' } });
+    assert.equal(c.schema.hash, 'ffffffff');
+    assert.ok(c.schema.empty);
+});
+
+test('a frame of another protocol version is reported', () => {
+    const { c, ws } = connected();
+    let mismatch = null;
+    c.on('versionMismatch', (d) => { mismatch = d; });
+    ws.deliver({ v: 3, t: 1, d: {} });
+    assert.deepEqual(mismatch, { device: 3, client: 2 });
+});
+
+test('a new token connects again with it', () => {
+    const { c, ws } = connected();
+    c.setToken('new');
+    assert.ok(ws.closed);
+    c.timers.runAll();
+    assert.match(FakeSocket.last.url, /apikey=new/);
+});
+
+test('subscribe sends the feed number', () => {
+    const { c, ws } = connected();
+    c.subscribe('frames');
+    const sent = ws.sent[ws.sent.length - 1];
+    assert.equal(sent.t, T.subscribe);
+    assert.deepEqual(sent.d, { fe: schema.feedNumbers.frames });
+});

test/js/helpers.js

@@ -0,0 +1,82 @@
+// What the web UI client tests share: the frames the e2e suite recorded from a
+// device (test/fixtures/proto2), and stand-ins for the socket and the timers.
+const path = require('node:path');
+
+const FIXTURES = path.join(__dirname, '..', 'fixtures', 'proto2');
+const hello = require(path.join(FIXTURES, 'hello.json'));
+const sync = require(path.join(FIXTURES, 'sync.json'));
+const schemaJson = require(path.join(FIXTURES, 'schema.json'));
+const { ProtoSchema } = require('../../data-src/js/schema.js');
+
+// A schema filled from the recorded sync, as a client builds it.
+function recordedSchema() {
+    const s = new ProtoSchema();
+    for (const f of sync) if (f.t === 4) s.add(f.d);
+    s.hash = hello.d.sh;
+    return s;
+}
+// The recorded sync's device sections, expanded, frame by frame.
+function deviceSections(schema) {
+    return sync.filter((f) => f.t === 4).map((f) => {
+        const d = {};
+        for (const k of Object.keys(f.d)) if (!['T', 'F', 'E', 'Q'].includes(k)) d[k] = f.d[k];
+        return schema.expand(d);
+    });
+}
+// The recorded sync's items under their short keys.
+function recordedItems(key) {
+    return sync.filter((f) => f.t === 4 && f.d[key]).flatMap((f) => f.d[key]);
+}
+
+// A WebSocket the test drives: it records what the client sends, and the test
+// opens, delivers and closes.
+class FakeSocket {
+    constructor(url) {
+        this.url = url;
+        this.sent = [];
+        this.closed = false;
+        FakeSocket.last = this;
+        FakeSocket.all.push(this);
+    }
+    send(text) { this.sent.push(JSON.parse(text)); }
+    close() {
+        if (this.closed) return;
+        this.closed = true;
+        if (this.onclose) this.onclose({});
+    }
+    open() { if (this.onopen) this.onopen({}); }
+    deliver(frame) { this.onmessage({ data: typeof frame === 'string' ? frame : JSON.stringify(frame) }); }
+    // The device closing it, or refusing it before it opened.
+    drop() { this.close(); }
+}
+FakeSocket.all = [];
+FakeSocket.reset = () => { FakeSocket.all = []; FakeSocket.last = null; };
+
+// Timers the test runs by hand.
+class FakeTimers {
+    constructor() {
+        this.timers = new Map();
+        this.next = 1;
+        this.setTimeout = (fn, ms) => {
+            const id = this.next++;
+            this.timers.set(id, { fn: fn, ms: ms });
+            return id;
+        };
+        this.clearTimeout = (id) => { this.timers.delete(id); };
+    }
+    // Runs every timer waiting, and returns their delays.
+    runAll() {
+        const due = [...this.timers.entries()];
+        this.timers.clear();
+        for (const [, t] of due) t.fn();
+        return due.map(([, t]) => t.ms);
+    }
+}
+// localStorage as far as the client uses it.
+class FakeStorage {
+    constructor() { this.items = {}; }
+    getItem(key) { return Object.prototype.hasOwnProperty.call(this.items, key) ? this.items[key] : null; }
+    setItem(key, value) { this.items[key] = String(value); }
+}
+
+module.exports = { hello, sync, schemaJson, recordedSchema, deviceSections, recordedItems, FakeSocket, FakeTimers, FakeStorage };

test/js/model.test.js

@@ -0,0 +1,106 @@
+// The web UI's ProtoModel (data-src/js/model.js): the recorded sync, then
+// events.
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const { ProtoModel } = require('../../data-src/js/model.js');
+const { recordedSchema, deviceSections, recordedItems } = require('./helpers.js');
+
+function synced() {
+    const schema = recordedSchema();
+    const model = new ProtoModel(schema);
+    model.beginSync();
+    for (const d of deviceSections(schema)) model.applySync(d);
+    model.endSync();
+    return model;
+}
+
+test('the sync fills every room, shade and group', () => {
+    const m = synced();
+    assert.equal(m.rooms.size, recordedItems('R').length);
+    assert.equal(m.shades.size, recordedItems('S').length);
+    assert.equal(m.groups.size, recordedItems('G').length);
+    assert.equal(m.header.model, 'ESPSomfyRTS');
+    assert.equal(m.header.maxShades, 32);
+    assert.ok(m.status.freeHeap > 0);
+});
+
+test('a field the sync left out is its default', () => {
+    const m = synced();
+    const raw = recordedItems('S').find((s) => typeof s.m === 'undefined');
+    const shade = m.shades.get(raw.s);
+    // Left out because they were at their default.
+    assert.equal(shade.myPos, -1);
+    assert.equal(shade.stepSize, raw.z === undefined ? 100 : raw.z);
+    assert.equal(typeof shade.paired, 'boolean');
+    assert.ok(Array.isArray(shade.linkedRemotes));
+    for (const r of shade.linkedRemotes) assert.equal(typeof r.lastRollingCode, 'number');
+    for (const g of m.groupList()) assert.ok(Array.isArray(g.linkedShades));
+});
+
+test('the lists come in their sort order', () => {
+    const m = synced();
+    const orders = m.shadeList().map((s) => s.sortOrder);
+    assert.deepEqual(orders, [...orders].sort((a, b) => a - b));
+});
+
+test('a state event changes only the state, and a left-out field is its default', () => {
+    const m = synced();
+    const shade = m.shadeList()[0];
+    const name = shade.name;
+    // A shade at rest at the top: position 0 is left out.
+    assert.ok(m.applyEvent('shadeState', { shadeId: shade.shadeId, target: 40, direction: 1 }));
+    assert.equal(m.shades.get(shade.shadeId).position, 0);
+    assert.equal(m.shades.get(shade.shadeId).target, 40);
+    assert.equal(m.shades.get(shade.shadeId).name, name);
+    // Of a shade the client doesn't have yet: nothing to change.
+    assert.equal(m.applyEvent('shadeState', { shadeId: 250, position: 5 }), false);
+});
+
+test('a whole item replaces what was there', () => {
+    const m = synced();
+    const shade = m.shadeList()[0];
+    m.applyEvent('shade', { shadeId: shade.shadeId, name: 'Renamed' });
+    assert.equal(m.shades.get(shade.shadeId).name, 'Renamed');
+    assert.equal(m.shades.get(shade.shadeId).upTime, 0);
+    m.applyEvent('room', { roomId: 9, name: 'New' });
+    assert.equal(m.rooms.get(9).sortOrder, 0);
+});
+
+test('removals, orders and repeaters', () => {
+    const m = synced();
+    const changes = [];
+    m.onChange((kind, id) => changes.push([kind, id]));
+    const [first, second] = m.shadeList();
+    assert.ok(m.applyEvent('shadeRemoved', { shadeId: first.shadeId }));
+    assert.equal(m.shades.has(first.shadeId), false);
+    // An id it doesn't have is ignored.
+    assert.equal(m.applyEvent('roomRemoved', { roomId: 250 }), false);
+    m.applyEvent('shadeOrder', { ids: [second.shadeId] });
+    assert.equal(m.shades.get(second.shadeId).sortOrder, 0);
+    m.applyEvent('repeaters', { repeaters: [7, 8] });
+    assert.deepEqual(m.repeaters, [7, 8]);
+    assert.deepEqual(changes.map((c) => c[0]), ['shadeRemoved', 'order', 'repeaters']);
+});
+
+test('a sync drops what it no longer lists, and keeps what an event added during it', () => {
+    const m = synced();
+    const schema = m.schema;
+    m.applyEvent('room', { roomId: 99, name: 'Gone' });
+    m.beginSync();
+    m.applyEvent('room', { roomId: 98, name: 'Added during the sync' });
+    for (const d of deviceSections(schema)) m.applySync(d);
+    m.endSync();
+    assert.equal(m.rooms.has(99), false);
+    assert.equal(m.rooms.has(98), true);
+});
+
+test('status events', () => {
+    const m = synced();
+    m.applyEvent('memory', { freeHeap: 1000, uptime: 5 });
+    assert.equal(m.status.freeHeap, 1000);
+    assert.equal(m.status.loopRate, 0);
+    m.applyEvent('wifi', { strength: -60, channel: 6 });
+    assert.equal(m.status.ssid, '');
+    assert.equal(m.status.strength, -60);
+    assert.equal(m.applyEvent('frame', { remoteAddress: 1 }), false);
+});

test/js/schema.test.js

@@ -0,0 +1,56 @@
+// The web UI's ProtoSchema (data-src/js/schema.js) against the schema a
+// device sent (test/fixtures/proto2).
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const { ProtoSchema } = require('../../data-src/js/schema.js');
+const { recordedSchema, schemaJson } = require('./helpers.js');
+
+test('the schema is collected from every sync frame it spans', () => {
+    const s = recordedSchema();
+    assert.equal(Object.keys(s.types).length, Object.keys(schemaJson.types).length);
+    assert.equal(Object.keys(s.fields).length, Object.keys(schemaJson.fields).length);
+    assert.equal(s.typeNumbers.shadeCommand, 16);
+    assert.equal(s.typeNumbers.saveSettings, 50);
+    assert.equal(s.keys.shadeId, 's');
+    assert.equal(s.errorNumbers.busy, 5);
+    assert.equal(s.feedNumbers.frequencyScan, 2);
+});
+
+test('short keys become names, and 1/0 booleans, all the way down', () => {
+    const s = recordedSchema();
+    const shade = s.expand({ s: 3, n: 'Office', i: 1, fp: 0, lk: [{ a: 5, c: 9 }] });
+    assert.deepEqual(shade, { shadeId: 3, name: 'Office', paired: true, flipPosition: false,
+        linkedRemotes: [{ remoteAddress: 5, lastRollingCode: 9 }] });
+    // A whole number that happens to be 1 stays a number.
+    assert.deepEqual(s.expand({ p: 1 }), { position: 1 });
+});
+
+test('names become short keys for an op', () => {
+    const s = recordedSchema();
+    assert.deepEqual(s.pack({ shadeId: 3, command: 'Up', paired: true, sunny: false }), { s: 3, cm: 'Up', i: 1, sn: 0 });
+    assert.deepEqual(s.pack({ ids: [3, 1, 2] }), { is: [3, 1, 2] });
+    // Left out, not sent as undefined.
+    assert.deepEqual(s.pack({ shadeId: 3, target: undefined }), { s: 3 });
+    assert.throws(() => s.pack({ nonsense: 1 }), /no field nonsense/);
+});
+
+test('a field left out is its default', () => {
+    const s = recordedSchema();
+    assert.equal(s.defaultOf('myPos'), -1);
+    assert.equal(s.defaultOf('stepSize'), 100);
+    assert.equal(s.defaultOf('bitLength'), 56);
+    assert.equal(s.defaultOf('paired'), false);
+    assert.equal(s.defaultOf('name'), '');
+    assert.deepEqual(s.defaultOf('linkedShades'), []);
+    assert.deepEqual(s.fill({ shadeId: 2 }, ['shadeId', 'position', 'myPos']), { shadeId: 2, position: 0, myPos: -1 });
+});
+
+test('a cached schema comes back whole, with its hash', () => {
+    const s = recordedSchema();
+    const back = ProtoSchema.load(JSON.parse(JSON.stringify(s.save())));
+    assert.equal(back.hash, s.hash);
+    assert.deepEqual(back.fields, s.fields);
+    assert.deepEqual(back.types, s.types);
+    assert.equal(ProtoSchema.load(null), null);
+    assert.equal(ProtoSchema.load({ hash: 'x', rows: {} }), null);
+});