From 99c6019431adcb638f7a70924525b830867087ef Mon Sep 17 00:00:00 2001 From: cjkas Date: Sun, 25 Oct 2026 15:00:23 +0100 Subject: [PATCH 1/7] feat: web UI protocol 2 client layer, JS and CSS bundles (step 2a) - data-src/js: ProtoSchema (schema from the sync, short keys and names, defaults, cached by hash), ProtoModel (rooms, shades, groups, repeaters, header and status from the sync and the events) and ProtoClient (/ws, hello, sync with next, op() as a Promise of its reply, events, feeds, reconnect). Nothing uses them yet: 2b moves the UI onto them. - minify.py joins js/*.js and index.js into index.js, and the three CSS files into app.css, so a page load opens fewer connections. - test/js: node --test on the three modules, replaying the recorded fixtures; a CI job runs it and the firmware build waits for it. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/build.yaml | 19 ++- AGENTS.md | 1 + data-src/index.html | 4 +- data-src/js/client.js | 242 +++++++++++++++++++++++++++++++++++ data-src/js/model.js | 150 ++++++++++++++++++++++ data-src/js/schema.js | 96 ++++++++++++++ data-src/login.html | 4 +- minify.py | 52 ++++++-- test/e2e/test_survival.py | 2 +- test/js/client.test.js | 200 +++++++++++++++++++++++++++++ test/js/helpers.js | 82 ++++++++++++ test/js/model.test.js | 106 +++++++++++++++ test/js/schema.test.js | 56 ++++++++ 13 files changed, 994 insertions(+), 20 deletions(-) create mode 100644 data-src/js/client.js create mode 100644 data-src/js/model.js create mode 100644 data-src/js/schema.js create mode 100644 test/js/client.test.js create mode 100644 test/js/helpers.js create mode 100644 test/js/model.test.js create mode 100644 test/js/schema.test.js diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index b94857f..562aa41 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -28,9 +28,26 @@ jobs: - name: Run native unit tests run: pio test -e native + webui: + name: Web UI tests + runs-on: ubuntu-latest + + steps: + - name: Check out code + uses: actions/checkout@v4 + + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: "22" + + # The protocol 2 client (data-src/js) against the recorded fixtures. + - name: Run web UI tests + run: node --test "test/js/*.test.js" + build: # Don't build or publish firmware that fails its tests. - needs: test + needs: [test, webui] permissions: write-all name: ${{ matrix.name }} runs-on: ubuntu-latest diff --git a/AGENTS.md b/AGENTS.md index c4089ab..55a0dbf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,6 +2,7 @@ - Fix any test or type errors until the whole suite is green. - Add or update tests for the code you change, even if nobody asked. - to test run `pio test -e native` +- the web UI's protocol 2 client (`data-src/js/`): `node --test "test/js/*.test.js"`. The minify step joins those files into `index.js`, so each one ends with a `module.exports` footer for Node and adds nothing global but its classes. ## Building - Firmware builds need pioarduino's Core (`pip install pioarduino`, or the pioarduino IDE extension). Stock PlatformIO Core refuses the pinned platform, see [ADR 0003](docs/adr/0003-pioarduino-idf55.md). diff --git a/data-src/index.html b/data-src/index.html index d52572d..398f0f0 100644 --- a/data-src/index.html +++ b/data-src/index.html @@ -8,9 +8,7 @@ - - - + diff --git a/data-src/js/client.js b/data-src/js/client.js new file mode 100644 index 0000000..9adb85b --- /dev/null +++ b/data-src/js/client.js @@ -0,0 +1,242 @@ +// A socket protocol 2 client (docs/PROTOCOL.md): ws://host/ws, hello, the sync +// with next after each frame, ops answered by their reply, events and feeds. +// It keeps the shade model (ProtoModel) up to date and connects again when the +// socket closes. The WebSocket, the timers and the schema's storage can be +// handed in, so the tests (test/js) run it without a device. The minify step +// joins this file into index.js after schema.js and model.js. +class ProtoError extends Error { + // code is an error name from the schema (badRequest, busy ...), or + // notConnected, closed or timeout from the client itself. + constructor(code, desc, data) { + super(desc || code); + this.code = code; + this.desc = desc || code; + this.data = data || {}; + } +} +class ProtoClient { + // opts: host, token, WebSocket, storage (getItem/setItem, for the schema), + // setTimeout/clearTimeout, opTimeoutMs. + constructor(opts) { + const deps = ProtoClient.deps(); + this.Schema = deps.ProtoSchema; + this.host = opts.host; + this.token = opts.token || ''; + this.WebSocket = opts.WebSocket || (typeof WebSocket !== 'undefined' ? WebSocket : null); + this.storage = opts.storage || null; + this.setTimeout = opts.setTimeout || ((fn, ms) => setTimeout(fn, ms)); + this.clearTimeout = opts.clearTimeout || ((t) => clearTimeout(t)); + this.opTimeoutMs = opts.opTimeoutMs || 10000; + this.schema = this.loadSchema() || new this.Schema(); + this.model = new deps.ProtoModel(this.schema); + this.model.onChange((kind, id, item) => this.emit('change', { kind: kind, id: id, item: item })); + this.listeners = {}; + this.pending = new Map(); + this.nextId = 1; + this.ws = null; + this.ready = false; + this.opened = false; + this.stopped = true; + this.retries = 0; + this.reconnectTimer = null; + this.held = []; + this.fw = null; + } + // In the browser the classes are the ones index.js was joined with; in + // Node they are loaded. + static deps() { + if (typeof require === 'function' && typeof module !== 'undefined' && module.exports) + return { ProtoSchema: require('./schema.js').ProtoSchema, ProtoModel: require('./model.js').ProtoModel }; + return { ProtoSchema: ProtoSchema, ProtoModel: ProtoModel }; + } + + // cb(d) for a device event by its name (shadeState, frame ...), and for + // the client's own: open, hello, synced, change, refused (closed before it + // opened: the token may have been refused), closed, versionMismatch and + // error (a refusal the device couldn't tie to an op). + on(name, cb) { (this.listeners[name] = this.listeners[name] || []).push(cb); } + emit(name, d) { for (const cb of this.listeners[name] || []) cb(d); } + + connect() { + this.stopped = false; + this.open(); + } + close() { + this.stopped = true; + if (this.reconnectTimer) this.clearTimeout(this.reconnectTimer); + this.reconnectTimer = null; + if (this.ws) this.ws.close(); + } + // A new token, from a login: the device checks it as the socket opens. + setToken(token) { + this.token = token || ''; + if (this.ws) this.ws.close(); + else if (!this.stopped) this.open(); + } + url() { + const params = []; + if (this.token) params.push(`apikey=${encodeURIComponent(this.token)}`); + if (this.schema.hash) params.push(`schema=${encodeURIComponent(this.schema.hash)}`); + return `ws://${this.host}/ws${params.length ? '?' + params.join('&') : ''}`; + } + open() { + this.reconnectTimer = null; + this.ready = false; + this.opened = false; + this.held = []; + const ws = new this.WebSocket(this.url()); + this.ws = ws; + ws.onopen = () => { + this.opened = true; + this.retries = 0; + this.emit('open'); + }; + ws.onmessage = (evt) => this.receive(evt.data); + ws.onclose = () => { + if (this.ws !== ws) return; + this.ws = null; + this.ready = false; + for (const [, p] of this.pending) { + this.clearTimeout(p.timer); + p.reject(new ProtoError('closed', 'The connection closed')); + } + this.pending.clear(); + this.emit(this.opened ? 'closed' : 'refused'); + if (this.stopped) return; + // Soon after a close, slower after each failure, at most 10 s. + const ms = Math.min(500 * Math.pow(2, this.retries++), 10000); + this.reconnectTimer = this.setTimeout(() => this.open(), ms); + }; + } + send(type, d, id) { + const frame = { v: ProtoClient.VERSION, t: this.schema.typeNumbers[type] }; + if (typeof frame.t === 'undefined') throw new ProtoError('unknownOp', `The device has no message ${type}`); + if (id) frame.id = id; + if (d) frame.d = d; + this.ws.send(JSON.stringify(frame)); + } + + // Sends an op, or a control message that wants a reply, by its name with + // its fields by name. Resolves with the reply's fields, rejects with a + // ProtoError. + op(name, fields) { + return new Promise((resolve, reject) => { + if (!this.ready || !this.ws) { + reject(new ProtoError('notConnected', 'Not connected to the device')); + return; + } + const id = this.nextId++; + try { + this.send(name, this.schema.pack(fields || {}), id); + } catch (err) { + reject(err instanceof ProtoError ? err : new ProtoError('badRequest', err.message)); + return; + } + const timer = this.setTimeout(() => { + this.pending.delete(id); + reject(new ProtoError('timeout', `No reply to ${name}`)); + }, this.opTimeoutMs); + this.pending.set(id, { resolve: resolve, reject: reject, timer: timer }); + }); + } + subscribe(feed) { return this.op('subscribe', { feed: this.schema.feedNumbers[feed] }); } + unsubscribe(feed) { return this.op('unsubscribe', { feed: this.schema.feedNumbers[feed] }); } + // The shade model and status again, without the schema. + resync() { + this.model.beginSync(); + return this.op('sync'); + } + + receive(text) { + let frame; + try { + frame = JSON.parse(text); + } catch (err) { + return; + } + if (frame.v !== ProtoClient.VERSION) { + this.emit('versionMismatch', { device: frame.v, client: ProtoClient.VERSION }); + return; + } + const d = frame.d || {}; + // hello, the sync and syncEnd have fixed numbers: they come before the + // schema. + switch (frame.t) { + case 1: this.hello(d); return; + case 4: this.syncFrame(d); return; + case 5: this.syncEnd(); return; + default: break; + } + if (!this.ready) this.held.push(frame); + else this.dispatch(frame); + } + hello(d) { + this.fw = d.fw; + // A schema cached for another firmware is no use: the device sends the + // one it has, as it didn't get its hash. + if (this.schema.hash !== d.sh) { + this.schema = new this.Schema(); + this.model.schema = this.schema; + } + this.schema.hash = d.sh; + this.model.beginSync(); + this.emit('hello', d); + } + syncFrame(d) { + this.schema.add(d); + const device = {}; + for (const key of Object.keys(d)) if (!['T', 'F', 'E', 'Q'].includes(key)) device[key] = d[key]; + // The schema's rows come first, so a frame with the device's sections + // has the whole of it. + if (Object.keys(device).length) this.model.applySync(this.schema.expand(device)); + this.ws.send(JSON.stringify({ v: ProtoClient.VERSION, t: 3 })); + } + syncEnd() { + this.model.endSync(); + if (!this.ready) { + this.ready = true; + this.saveSchema(); + const held = this.held; + this.held = []; + for (const frame of held) this.dispatch(frame); + } + this.emit('synced', this.model); + } + dispatch(frame) { + const d = this.schema.expand(frame.d || {}); + if (frame.t === 2) { + const p = typeof frame.id !== 'undefined' ? this.pending.get(frame.id) : null; + if (!p) { + if (!d.ok) this.emit('error', new ProtoError(this.schema.errors[d.code] || 'internal', d.desc, d)); + return; + } + this.pending.delete(frame.id); + this.clearTimeout(p.timer); + if (d.ok) p.resolve(d); + else p.reject(new ProtoError(this.schema.errors[d.code] || 'internal', d.desc, d)); + return; + } + const name = this.schema.types[frame.t]; + if (!name) return; + this.model.applyEvent(name, d); + this.emit(name, d); + } + + loadSchema() { + if (!this.storage) return null; + try { + return this.Schema.load(JSON.parse(this.storage.getItem(ProtoClient.SCHEMA_KEY))); + } catch (err) { + return null; + } + } + saveSchema() { + if (!this.storage) return; + try { + this.storage.setItem(ProtoClient.SCHEMA_KEY, JSON.stringify(this.schema.save())); + } catch (err) { /* Private windows may have no storage: the device sends the schema each time. */ } + } +} +ProtoClient.VERSION = 2; +ProtoClient.SCHEMA_KEY = 'espsomfyProtoSchema'; +if (typeof module !== 'undefined' && module.exports) module.exports = { ProtoClient, ProtoError }; diff --git a/data-src/js/model.js b/data-src/js/model.js new file mode 100644 index 0000000..2ab15e4 --- /dev/null +++ b/data-src/js/model.js @@ -0,0 +1,150 @@ +// The shade model as socket protocol 2 keeps a client up to date with it +// (docs/PROTOCOL.md): rooms, shades, groups and repeaters from the sync, then +// changed by the device's events. Items are inserted or updated by id, so an +// event during the sync does no harm. It takes frames already expanded to +// names (ProtoSchema.expand) and fills in what a frame leaves out at its +// default. The minify step joins this file into index.js; Node loads it on +// its own for the tests (test/js). +class ProtoModel { + constructor(schema) { + this.schema = schema; + this.rooms = new Map(); + this.shades = new Map(); + this.groups = new Map(); + this.repeaters = []; + this.header = {}; + this.status = {}; + this.listeners = []; + this.seen = null; + } + // cb(kind, id, item): kind is room, shade, group, roomRemoved, + // shadeRemoved, groupRemoved, order, repeaters, header or status. + onChange(cb) { this.listeners.push(cb); } + changed(kind, id, item) { for (const cb of this.listeners) cb(kind, id, item); } + + // A sync: the items it doesn't list are gone once it ends. + beginSync() { this.seen = { rooms: new Set(), shades: new Set(), groups: new Set() }; } + applySync(d) { + if (d.header) { + this.header = this.schema.fill(Object.assign({}, d.header), ProtoModel.FIELDS.header); + this.changed('header', 0, this.header); + } + for (const r of d.rooms || []) this.putRoom(r); + for (const s of d.shades || []) this.putShade(s); + for (const g of d.groups || []) this.putGroup(g); + if (d.repeaters) this.putRepeaters(d.repeaters); + if (d.status) { + this.putStatus(d.status, ProtoModel.FIELDS.memory); + this.putStatus(d.status, this.header.connType === 'Ethernet' ? ProtoModel.FIELDS.ethernet : ProtoModel.FIELDS.wifi); + } + } + endSync() { + if (!this.seen) return; + const lists = [['rooms', 'roomRemoved'], ['shades', 'shadeRemoved'], ['groups', 'groupRemoved']]; + for (const [list, removed] of lists) { + for (const id of [...this[list].keys()]) { + if (this.seen[list].has(id)) continue; + this[list].delete(id); + this.changed(removed, id); + } + } + this.seen = null; + } + // An event by its type's name; false for one that isn't about the model. + applyEvent(type, d) { + switch (type) { + case 'room': this.putRoom(d); return true; + case 'shade': this.putShade(d); return true; + case 'group': this.putGroup(d); return true; + case 'shadeState': return this.mergeState(this.shades, 'shade', d.shadeId, d, ProtoModel.FIELDS.shadeState); + case 'groupState': return this.mergeState(this.groups, 'group', d.groupId, d, ProtoModel.FIELDS.groupState); + case 'roomRemoved': return this.remove(this.rooms, type, d.roomId || 0); + case 'shadeRemoved': return this.remove(this.shades, type, d.shadeId || 0); + case 'groupRemoved': return this.remove(this.groups, type, d.groupId || 0); + case 'roomOrder': return this.order(this.rooms, d.ids || []); + case 'shadeOrder': return this.order(this.shades, d.ids || []); + case 'groupOrder': return this.order(this.groups, d.ids || []); + case 'repeaters': this.putRepeaters(d.repeaters || []); return true; + case 'memory': this.putStatus(d, ProtoModel.FIELDS.memory); return true; + case 'wifi': this.putStatus(d, ProtoModel.FIELDS.wifi); return true; + case 'ethernet': this.putStatus(d, ProtoModel.FIELDS.ethernet); return true; + default: return false; + } + } + + // The lists in their sort order. + roomList() { return ProtoModel.sorted(this.rooms); } + shadeList() { return ProtoModel.sorted(this.shades); } + groupList() { return ProtoModel.sorted(this.groups); } + static sorted(map) { return [...map.values()].sort((a, b) => a.sortOrder - b.sortOrder); } + + putRoom(r) { this.put(this.rooms, 'room', r.roomId, r, ProtoModel.FIELDS.room, 'rooms'); } + putShade(s) { + // Its linked remotes leave out their defaults too. + const linked = (s.linkedRemotes || []).map((r) => this.schema.fill(Object.assign({}, r), ProtoModel.FIELDS.linkedRemote)); + this.put(this.shades, 'shade', s.shadeId, Object.assign({}, s, { linkedRemotes: linked }), ProtoModel.FIELDS.shade, 'shades'); + } + putGroup(g) { this.put(this.groups, 'group', g.groupId, g, ProtoModel.FIELDS.group, 'groups'); } + // A whole item: what it had before is replaced. + put(map, kind, id, item, fields, list) { + id = id || 0; + const whole = this.schema.fill(Object.assign({}, item), fields); + map.set(id, whole); + if (this.seen) this.seen[list].add(id); + this.changed(kind, id, whole); + } + // Part of an item: only its state. A state of an item not synced yet + // waits for the item. + mergeState(map, kind, id, d, fields) { + const item = map.get(id || 0); + if (!item) return false; + Object.assign(item, this.schema.fill(Object.assign({}, d), fields)); + this.changed(kind, item[`${kind}Id`], item); + return true; + } + remove(map, kind, id) { + if (!map.delete(id)) return false; + this.changed(kind, id); + return true; + } + order(map, ids) { + ids.forEach((id, i) => { + const item = map.get(id); + if (item) item.sortOrder = i; + }); + this.changed('order', 0, ids); + return true; + } + putRepeaters(list) { + this.repeaters = list.slice(); + this.changed('repeaters', 0, this.repeaters); + } + putStatus(d, fields) { + const part = {}; + for (const name of fields) part[name] = typeof d[name] === 'undefined' ? this.schema.defaultOf(name) : d[name]; + Object.assign(this.status, part); + this.changed('status', 0, this.status); + } +} +// The fields of each kind of item, as the device writes them +// (src/ProtoSync.cpp, src/outputs/ProtoSocket.cpp): what a frame that leaves +// one out means by it. +ProtoModel.FIELDS = { + header: ['maxRooms', 'maxShades', 'maxGroups', 'maxGroupedShades', 'maxLinkedRemotes', 'startingAddress', 'bitLength', + 'proto', 'serverId', 'hostname', 'model', 'chipModel', 'fw', 'connType'], + room: ['roomId', 'name', 'sortOrder'], + shadeState: ['shadeId', 'position', 'target', 'direction', 'myPos', 'tiltPosition', 'tiltTarget', 'tiltDirection', + 'myTiltPos', 'flags', 'lastRollingCode'], + groupState: ['groupId', 'flags', 'direction'], + linkedRemote: ['remoteAddress', 'lastRollingCode'], + group: ['groupId', 'roomId', 'name', 'remoteAddress', 'lastRollingCode', 'bitLength', 'proto', 'sunSensor', + 'flipCommands', 'flags', 'repeats', 'sortOrder', 'linkedShades'], + memory: ['freeHeap', 'maxHeap', 'minHeap', 'totalHeap', 'uptime', 'loopRate', 'loopMaxUs'], + wifi: ['ssid', 'strength', 'channel'], + ethernet: ['connected', 'speed', 'fullDuplex'] +}; +ProtoModel.FIELDS.shade = ProtoModel.FIELDS.shadeState.concat(['roomId', 'name', 'remoteAddress', 'upTime', 'downTime', + 'tiltTime', 'stepSize', 'paired', 'tiltType', 'shadeType', 'bitLength', 'proto', 'flipCommands', 'flipPosition', + 'inGroup', 'sunSensor', 'light', 'simMy', 'repeats', 'sortOrder', 'gpioUp', 'gpioDown', 'gpioMy', 'gpioLLTrigger', + 'linkedRemotes']); +if (typeof module !== 'undefined' && module.exports) module.exports = { ProtoModel }; diff --git a/data-src/js/schema.js b/data-src/js/schema.js new file mode 100644 index 0000000..50e34ea --- /dev/null +++ b/data-src/js/schema.js @@ -0,0 +1,96 @@ +// Socket protocol 2's schema (docs/PROTOCOL.md): the device's table of message +// types, fields, error codes and feeds, as the sync sends it. It turns the +// short keys of a frame into names and back. The minify step joins this file +// into index.js; Node loads it on its own for the tests (test/js). +class ProtoSchema { + constructor() { + this.hash = null; + this.types = {}; // number -> name + this.typeNumbers = {}; // name -> number + this.fields = {}; // key -> { name, kind, def } + this.keys = {}; // name -> key + this.errors = {}; + this.errorNumbers = {}; + this.feeds = {}; + this.feedNumbers = {}; + this.rows = { T: [], F: [], E: [], Q: [] }; + } + // The schema's rows in a sync frame's payload: T types, F fields, E error + // codes, Q feeds. They may span several frames. + add(d) { + for (const [number, name] of d.T || []) { + this.types[number] = name; + this.typeNumbers[name] = number; + } + for (const [key, name, kind, def] of d.F || []) { + this.fields[key] = { name: name, kind: kind, def: def }; + this.keys[name] = key; + } + for (const [number, name] of d.E || []) { + this.errors[number] = name; + this.errorNumbers[name] = number; + } + for (const [number, name] of d.Q || []) { + this.feeds[number] = name; + this.feedNumbers[name] = number; + } + for (const k of ['T', 'F', 'E', 'Q']) if (d[k]) this.rows[k].push(...d[k]); + } + get empty() { return Object.keys(this.fields).length === 0; } + // What a field left out of a frame is: its default. + defaultOf(name) { + const f = this.fields[this.keys[name]]; + if (!f) return undefined; + switch (f.kind) { + case 'b': return f.def !== 0; + case 's': return ''; + case 'l': return []; + case 'o': return {}; + default: return f.def; + } + } + // Short keys to names, and 1/0 to true/false, all the way down. A field + // left out stays out; fill() puts the defaults in. + expand(value) { + if (Array.isArray(value)) return value.map((v) => this.expand(v)); + if (value === null || typeof value !== 'object') return value; + const out = {}; + for (const key of Object.keys(value)) { + const f = this.fields[key]; + let v = value[key]; + if (f && f.kind === 'b' && typeof v === 'number') v = v !== 0; + out[f ? f.name : key] = this.expand(v); + } + return out; + } + // Names to short keys, and booleans to 1/0, for an op's payload. A name + // the schema doesn't have is an error: the device would refuse it anyway. + pack(value) { + if (Array.isArray(value)) return value.map((v) => this.pack(v)); + if (value === null || typeof value !== 'object') return typeof value === 'boolean' ? (value ? 1 : 0) : value; + const out = {}; + for (const name of Object.keys(value)) { + if (typeof value[name] === 'undefined') continue; + const key = this.keys[name]; + if (typeof key === 'undefined') throw new Error(`The device has no field ${name}`); + out[key] = this.pack(value[name]); + } + return out; + } + // Sets each of names that obj lacks to its default, as a frame leaves out + // a field at its default. Returns obj. + fill(obj, names) { + for (const name of names) if (typeof obj[name] === 'undefined') obj[name] = this.defaultOf(name); + return obj; + } + // What a cache keeps, and the schema back from it. + save() { return { hash: this.hash, rows: this.rows }; } + static load(saved) { + if (!saved || typeof saved.hash !== 'string' || !saved.rows) return null; + const s = new ProtoSchema(); + s.add(saved.rows); + s.hash = saved.hash; + return s.empty ? null : s; + } +} +if (typeof module !== 'undefined' && module.exports) module.exports = { ProtoSchema }; diff --git a/data-src/login.html b/data-src/login.html index 32623f1..c789f6c 100644 --- a/data-src/login.html +++ b/data-src/login.html @@ -3,9 +3,7 @@ - - - + diff --git a/minify.py b/minify.py index a5c581c..f7b48d6 100644 --- a/minify.py +++ b/minify.py @@ -52,6 +52,16 @@ MINIFY_AND_GZIP = {".html", ".htm", ".css", ".js", ".json", ".svg", ".xml"} # Extensions that optipng can optimise PNG_EXTENSIONS = {".png"} +# Files joined into one, in this order, before they are minified: a page load +# then opens fewer connections, each of which costs the device heap +# (docs/adr/0005-one-websocket-protocol.md). The protocol 2 client in js/ +# comes before index.js, which uses it. A source listed here isn't copied on +# its own. +BUNDLES = { + "index.js": ["js/schema.js", "js/model.js", "js/client.js", "index.js"], + "app.css": ["main.css", "widgets.css", "icons.css"], +} + # Everything else is copied verbatim # ────────────────────────────────────────────── @@ -252,6 +262,33 @@ def minify_all(): total_final = 0 file_count = 0 + def report(rel_path, stats): + nonlocal total_original, total_final, file_count + pct = (1 - stats["final"] / stats["original"]) * 100 if stats["original"] > 0 else 0 + print( + f" {rel_path:<40s} " + f"{stats['original']:>8,d} → {stats['final']:>8,d} B " + f"({pct:+.0f}%) [{stats['action']}]" + ) + total_original += stats["original"] + total_final += stats["final"] + file_count += 1 + + bundled = set() + work_dir = os.path.join(dst_dir, ".bundles") + os.makedirs(work_dir, exist_ok=True) + for name, parts in BUNDLES.items(): + joined = os.path.join(work_dir, name) + with open(joined, "w", encoding="utf-8") as out: + for part in parts: + with open(os.path.join(src_dir, part), "r", encoding="utf-8") as f: + out.write(f.read()) + # A file without a final newline would run into the next. + out.write("\n") + bundled.add(os.path.normpath(part)) + report(f"{name} ({len(parts)} files)", process_file(joined, os.path.join(dst_dir, name))) + shutil.rmtree(work_dir) + for root, dirs, files in os.walk(src_dir): for fname in sorted(files): # Skip hidden files and editor temp files @@ -260,20 +297,11 @@ def minify_all(): src_path = os.path.join(root, fname) rel_path = os.path.relpath(src_path, src_dir) + if os.path.normpath(rel_path) in bundled: + continue dst_path = os.path.join(dst_dir, rel_path) - stats = process_file(src_path, dst_path) - - pct = (1 - stats["final"] / stats["original"]) * 100 if stats["original"] > 0 else 0 - print( - f" {rel_path:<40s} " - f"{stats['original']:>8,d} → {stats['final']:>8,d} B " - f"({pct:+.0f}%) [{stats['action']}]" - ) - - total_original += stats["original"] - total_final += stats["final"] - file_count += 1 + report(rel_path, process_file(src_path, dst_path)) print(f"[minify] {file_count} files processed") print( diff --git a/test/e2e/test_survival.py b/test/e2e/test_survival.py index 457e40e..a6dcae9 100644 --- a/test/e2e/test_survival.py +++ b/test/e2e/test_survival.py @@ -54,7 +54,7 @@ def test_one_client_too_many_is_closed(connect, schema): def test_page_load(host, token, connect, watch_heap): """The static files in parallel, as a browser fetches them, then the socket.""" - files = ["/", "/main.css", "/widgets.css", "/icons.css", "/index.js", "/icon.png"] + files = ["/", "/app.css", "/index.js", "/icon.png"] errors = [] def get(path): diff --git a/test/js/client.test.js b/test/js/client.test.js new file mode 100644 index 0000000..17d5c39 --- /dev/null +++ b/test/js/client.test.js @@ -0,0 +1,200 @@ +// The web UI's ProtoClient (data-src/js/client.js): the recorded hello and +// sync replayed through a stand-in socket. +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { ProtoClient, ProtoError } = require('../../data-src/js/client.js'); +const { hello, sync, recordedItems, recordedSchema, FakeSocket, FakeTimers, FakeStorage } = require('./helpers.js'); + +function client(opts) { + FakeSocket.reset(); + const timers = new FakeTimers(); + const c = new ProtoClient(Object.assign({ host: 'device', WebSocket: FakeSocket, setTimeout: timers.setTimeout, + clearTimeout: timers.clearTimeout }, opts || {})); + c.timers = timers; + return c; +} +// Connects and plays the recorded hello and sync. +function connected(opts) { + const c = client(opts); + c.connect(); + const ws = FakeSocket.last; + ws.open(); + ws.deliver(hello); + for (const f of sync) ws.deliver(f); + return { c, ws }; +} +const schema = recordedSchema(); +const T = schema.typeNumbers; +function reply(id, d) { return { v: 2, t: T.reply, id: id, d: d }; } + +test('the client connects with its token and syncs, sending next after each sync frame', () => { + const c = client({ token: 'abc 123' }); + c.connect(); + const ws = FakeSocket.last; + assert.equal(ws.url, 'ws://device/ws?apikey=abc%20123'); + ws.open(); + ws.deliver(hello); + for (const f of sync) ws.deliver(f); + const syncFrames = sync.filter((f) => f.t === T.sync).length; + assert.equal(ws.sent.filter((f) => f.t === T.next).length, syncFrames); + assert.ok(c.ready); + assert.equal(c.fw, hello.d.fw); + assert.equal(c.model.shades.size, recordedItems('S').length); +}); + +test('synced comes once the sync has ended', () => { + const c = client(); + let synced = 0; + c.on('synced', () => synced++); + c.connect(); + const ws = FakeSocket.last; + ws.open(); + ws.deliver(hello); + for (const f of sync.filter((f) => f.t === T.sync)) ws.deliver(f); + assert.equal(synced, 0); + ws.deliver(sync.find((f) => f.t === T.syncEnd)); + assert.equal(synced, 1); +}); + +test('an op resolves with its reply', async () => { + const { c, ws } = connected(); + const p = c.op('getSettings', { section: 'radio' }); + const sent = ws.sent[ws.sent.length - 1]; + assert.deepEqual(sent, { v: 2, t: T.getSettings, id: sent.id, d: { se: 'radio' } }); + ws.deliver(reply(sent.id, { k: 1, bl: 80, en: 1 })); + assert.deepEqual(await p, { ok: true, bitLength: 80, enabled: true }); +}); + +test('a refused op rejects with the error by its name', async () => { + const { c, ws } = connected(); + const p = c.op('saveSettings', { section: 'mqtt' }); + const id = ws.sent[ws.sent.length - 1].id; + ws.deliver(reply(id, { e: schema.errorNumbers.badRequest, x: 'Nothing to save' })); + await assert.rejects(p, (err) => err instanceof ProtoError && err.code === 'badRequest' && err.desc === 'Nothing to save'); +}); + +test('an op is refused before the sync, and when its reply never comes', async () => { + const c = client(); + await assert.rejects(c.op('reboot'), (err) => err.code === 'notConnected'); + const { c: c2 } = connected(); + const p = c2.op('reboot'); + c2.timers.runAll(); + await assert.rejects(p, (err) => err.code === 'timeout'); +}); + +test('an op of a name or field the device lacks is refused here', async () => { + const { c } = connected(); + await assert.rejects(c.op('noSuchOp'), (err) => err.code === 'unknownOp'); + await assert.rejects(c.op('reboot', { noSuchField: 1 }), (err) => err.code === 'badRequest'); +}); + +test('ops waiting when the socket closes are refused, and the client connects again', async () => { + const { c, ws } = connected(); + const p = c.op('scanWifi'); + let closed = 0; + c.on('closed', () => closed++); + ws.drop(); + await assert.rejects(p, (err) => err.code === 'closed'); + assert.equal(closed, 1); + assert.equal(c.ready, false); + assert.deepEqual(c.timers.runAll(), [500]); + assert.notEqual(FakeSocket.last, ws); +}); + +test('a socket refused before it opened says so, and the retries slow down', () => { + const c = client(); + let refused = 0; + c.on('refused', () => refused++); + c.connect(); + const delays = []; + for (let i = 0; i < 6; i++) { + FakeSocket.last.drop(); + delays.push(...c.timers.runAll()); + } + assert.equal(refused, 6); + assert.deepEqual(delays, [500, 1000, 2000, 4000, 8000, 10000]); +}); + +test('close stops the reconnecting', () => { + const { c } = connected(); + c.close(); + assert.deepEqual(c.timers.runAll(), []); +}); + +test('events update the model, and are told by name', () => { + const { c, ws } = connected(); + const shade = c.model.shadeList()[0]; + const seen = []; + c.on('shadeState', (d) => seen.push(d)); + ws.deliver({ v: 2, t: T.shadeState, d: { s: shade.shadeId, t: 30, d: 1 } }); + assert.equal(c.model.shades.get(shade.shadeId).target, 30); + assert.deepEqual(seen, [{ shadeId: shade.shadeId, target: 30, direction: 1 }]); +}); + +test('an event before the schema is kept until the sync ends', () => { + const c = client(); + c.connect(); + const ws = FakeSocket.last; + ws.open(); + ws.deliver(hello); + // Before the schema: the client can't read it yet. + const shadeId = recordedItems('S')[0].s; + ws.deliver({ v: 2, t: T.shadeState, d: { s: shadeId, t: 55 } }); + for (const f of sync) ws.deliver(f); + assert.equal(c.model.shades.get(shadeId).target, 55); +}); + +test('the schema is kept, and the device is told it has it', () => { + const storage = new FakeStorage(); + connected({ storage: storage }); + const c = client({ storage: storage }); + c.connect(); + const ws = FakeSocket.last; + assert.equal(ws.url, `ws://device/ws?schema=${hello.d.sh}`); + ws.open(); + ws.deliver(hello); + // Without the schema's rows, as the device sends a client that has it. + for (const f of sync) { + const d = Object.assign({}, f.d); + for (const k of ['T', 'F', 'E', 'Q']) delete d[k]; + ws.deliver(Object.assign({}, f, { d: d })); + } + assert.ok(c.ready); + assert.equal(c.model.shades.size, recordedItems('S').length); +}); + +test('a kept schema of another firmware is dropped', () => { + const storage = new FakeStorage(); + connected({ storage: storage }); + const c = client({ storage: storage }); + c.connect(); + const ws = FakeSocket.last; + ws.open(); + ws.deliver({ v: 2, t: 1, d: { fw: 'v4.0.0', sh: 'ffffffff' } }); + assert.equal(c.schema.hash, 'ffffffff'); + assert.ok(c.schema.empty); +}); + +test('a frame of another protocol version is reported', () => { + const { c, ws } = connected(); + let mismatch = null; + c.on('versionMismatch', (d) => { mismatch = d; }); + ws.deliver({ v: 3, t: 1, d: {} }); + assert.deepEqual(mismatch, { device: 3, client: 2 }); +}); + +test('a new token connects again with it', () => { + const { c, ws } = connected(); + c.setToken('new'); + assert.ok(ws.closed); + c.timers.runAll(); + assert.match(FakeSocket.last.url, /apikey=new/); +}); + +test('subscribe sends the feed number', () => { + const { c, ws } = connected(); + c.subscribe('frames'); + const sent = ws.sent[ws.sent.length - 1]; + assert.equal(sent.t, T.subscribe); + assert.deepEqual(sent.d, { fe: schema.feedNumbers.frames }); +}); diff --git a/test/js/helpers.js b/test/js/helpers.js new file mode 100644 index 0000000..fd73718 --- /dev/null +++ b/test/js/helpers.js @@ -0,0 +1,82 @@ +// What the web UI client tests share: the frames the e2e suite recorded from a +// device (test/fixtures/proto2), and stand-ins for the socket and the timers. +const path = require('node:path'); + +const FIXTURES = path.join(__dirname, '..', 'fixtures', 'proto2'); +const hello = require(path.join(FIXTURES, 'hello.json')); +const sync = require(path.join(FIXTURES, 'sync.json')); +const schemaJson = require(path.join(FIXTURES, 'schema.json')); +const { ProtoSchema } = require('../../data-src/js/schema.js'); + +// A schema filled from the recorded sync, as a client builds it. +function recordedSchema() { + const s = new ProtoSchema(); + for (const f of sync) if (f.t === 4) s.add(f.d); + s.hash = hello.d.sh; + return s; +} +// The recorded sync's device sections, expanded, frame by frame. +function deviceSections(schema) { + return sync.filter((f) => f.t === 4).map((f) => { + const d = {}; + for (const k of Object.keys(f.d)) if (!['T', 'F', 'E', 'Q'].includes(k)) d[k] = f.d[k]; + return schema.expand(d); + }); +} +// The recorded sync's items under their short keys. +function recordedItems(key) { + return sync.filter((f) => f.t === 4 && f.d[key]).flatMap((f) => f.d[key]); +} + +// A WebSocket the test drives: it records what the client sends, and the test +// opens, delivers and closes. +class FakeSocket { + constructor(url) { + this.url = url; + this.sent = []; + this.closed = false; + FakeSocket.last = this; + FakeSocket.all.push(this); + } + send(text) { this.sent.push(JSON.parse(text)); } + close() { + if (this.closed) return; + this.closed = true; + if (this.onclose) this.onclose({}); + } + open() { if (this.onopen) this.onopen({}); } + deliver(frame) { this.onmessage({ data: typeof frame === 'string' ? frame : JSON.stringify(frame) }); } + // The device closing it, or refusing it before it opened. + drop() { this.close(); } +} +FakeSocket.all = []; +FakeSocket.reset = () => { FakeSocket.all = []; FakeSocket.last = null; }; + +// Timers the test runs by hand. +class FakeTimers { + constructor() { + this.timers = new Map(); + this.next = 1; + this.setTimeout = (fn, ms) => { + const id = this.next++; + this.timers.set(id, { fn: fn, ms: ms }); + return id; + }; + this.clearTimeout = (id) => { this.timers.delete(id); }; + } + // Runs every timer waiting, and returns their delays. + runAll() { + const due = [...this.timers.entries()]; + this.timers.clear(); + for (const [, t] of due) t.fn(); + return due.map(([, t]) => t.ms); + } +} +// localStorage as far as the client uses it. +class FakeStorage { + constructor() { this.items = {}; } + getItem(key) { return Object.prototype.hasOwnProperty.call(this.items, key) ? this.items[key] : null; } + setItem(key, value) { this.items[key] = String(value); } +} + +module.exports = { hello, sync, schemaJson, recordedSchema, deviceSections, recordedItems, FakeSocket, FakeTimers, FakeStorage }; diff --git a/test/js/model.test.js b/test/js/model.test.js new file mode 100644 index 0000000..a7dc239 --- /dev/null +++ b/test/js/model.test.js @@ -0,0 +1,106 @@ +// The web UI's ProtoModel (data-src/js/model.js): the recorded sync, then +// events. +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { ProtoModel } = require('../../data-src/js/model.js'); +const { recordedSchema, deviceSections, recordedItems } = require('./helpers.js'); + +function synced() { + const schema = recordedSchema(); + const model = new ProtoModel(schema); + model.beginSync(); + for (const d of deviceSections(schema)) model.applySync(d); + model.endSync(); + return model; +} + +test('the sync fills every room, shade and group', () => { + const m = synced(); + assert.equal(m.rooms.size, recordedItems('R').length); + assert.equal(m.shades.size, recordedItems('S').length); + assert.equal(m.groups.size, recordedItems('G').length); + assert.equal(m.header.model, 'ESPSomfyRTS'); + assert.equal(m.header.maxShades, 32); + assert.ok(m.status.freeHeap > 0); +}); + +test('a field the sync left out is its default', () => { + const m = synced(); + const raw = recordedItems('S').find((s) => typeof s.m === 'undefined'); + const shade = m.shades.get(raw.s); + // Left out because they were at their default. + assert.equal(shade.myPos, -1); + assert.equal(shade.stepSize, raw.z === undefined ? 100 : raw.z); + assert.equal(typeof shade.paired, 'boolean'); + assert.ok(Array.isArray(shade.linkedRemotes)); + for (const r of shade.linkedRemotes) assert.equal(typeof r.lastRollingCode, 'number'); + for (const g of m.groupList()) assert.ok(Array.isArray(g.linkedShades)); +}); + +test('the lists come in their sort order', () => { + const m = synced(); + const orders = m.shadeList().map((s) => s.sortOrder); + assert.deepEqual(orders, [...orders].sort((a, b) => a - b)); +}); + +test('a state event changes only the state, and a left-out field is its default', () => { + const m = synced(); + const shade = m.shadeList()[0]; + const name = shade.name; + // A shade at rest at the top: position 0 is left out. + assert.ok(m.applyEvent('shadeState', { shadeId: shade.shadeId, target: 40, direction: 1 })); + assert.equal(m.shades.get(shade.shadeId).position, 0); + assert.equal(m.shades.get(shade.shadeId).target, 40); + assert.equal(m.shades.get(shade.shadeId).name, name); + // Of a shade the client doesn't have yet: nothing to change. + assert.equal(m.applyEvent('shadeState', { shadeId: 250, position: 5 }), false); +}); + +test('a whole item replaces what was there', () => { + const m = synced(); + const shade = m.shadeList()[0]; + m.applyEvent('shade', { shadeId: shade.shadeId, name: 'Renamed' }); + assert.equal(m.shades.get(shade.shadeId).name, 'Renamed'); + assert.equal(m.shades.get(shade.shadeId).upTime, 0); + m.applyEvent('room', { roomId: 9, name: 'New' }); + assert.equal(m.rooms.get(9).sortOrder, 0); +}); + +test('removals, orders and repeaters', () => { + const m = synced(); + const changes = []; + m.onChange((kind, id) => changes.push([kind, id])); + const [first, second] = m.shadeList(); + assert.ok(m.applyEvent('shadeRemoved', { shadeId: first.shadeId })); + assert.equal(m.shades.has(first.shadeId), false); + // An id it doesn't have is ignored. + assert.equal(m.applyEvent('roomRemoved', { roomId: 250 }), false); + m.applyEvent('shadeOrder', { ids: [second.shadeId] }); + assert.equal(m.shades.get(second.shadeId).sortOrder, 0); + m.applyEvent('repeaters', { repeaters: [7, 8] }); + assert.deepEqual(m.repeaters, [7, 8]); + assert.deepEqual(changes.map((c) => c[0]), ['shadeRemoved', 'order', 'repeaters']); +}); + +test('a sync drops what it no longer lists, and keeps what an event added during it', () => { + const m = synced(); + const schema = m.schema; + m.applyEvent('room', { roomId: 99, name: 'Gone' }); + m.beginSync(); + m.applyEvent('room', { roomId: 98, name: 'Added during the sync' }); + for (const d of deviceSections(schema)) m.applySync(d); + m.endSync(); + assert.equal(m.rooms.has(99), false); + assert.equal(m.rooms.has(98), true); +}); + +test('status events', () => { + const m = synced(); + m.applyEvent('memory', { freeHeap: 1000, uptime: 5 }); + assert.equal(m.status.freeHeap, 1000); + assert.equal(m.status.loopRate, 0); + m.applyEvent('wifi', { strength: -60, channel: 6 }); + assert.equal(m.status.ssid, ''); + assert.equal(m.status.strength, -60); + assert.equal(m.applyEvent('frame', { remoteAddress: 1 }), false); +}); diff --git a/test/js/schema.test.js b/test/js/schema.test.js new file mode 100644 index 0000000..0ec1136 --- /dev/null +++ b/test/js/schema.test.js @@ -0,0 +1,56 @@ +// The web UI's ProtoSchema (data-src/js/schema.js) against the schema a +// device sent (test/fixtures/proto2). +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { ProtoSchema } = require('../../data-src/js/schema.js'); +const { recordedSchema, schemaJson } = require('./helpers.js'); + +test('the schema is collected from every sync frame it spans', () => { + const s = recordedSchema(); + assert.equal(Object.keys(s.types).length, Object.keys(schemaJson.types).length); + assert.equal(Object.keys(s.fields).length, Object.keys(schemaJson.fields).length); + assert.equal(s.typeNumbers.shadeCommand, 16); + assert.equal(s.typeNumbers.saveSettings, 50); + assert.equal(s.keys.shadeId, 's'); + assert.equal(s.errorNumbers.busy, 5); + assert.equal(s.feedNumbers.frequencyScan, 2); +}); + +test('short keys become names, and 1/0 booleans, all the way down', () => { + const s = recordedSchema(); + const shade = s.expand({ s: 3, n: 'Office', i: 1, fp: 0, lk: [{ a: 5, c: 9 }] }); + assert.deepEqual(shade, { shadeId: 3, name: 'Office', paired: true, flipPosition: false, + linkedRemotes: [{ remoteAddress: 5, lastRollingCode: 9 }] }); + // A whole number that happens to be 1 stays a number. + assert.deepEqual(s.expand({ p: 1 }), { position: 1 }); +}); + +test('names become short keys for an op', () => { + const s = recordedSchema(); + assert.deepEqual(s.pack({ shadeId: 3, command: 'Up', paired: true, sunny: false }), { s: 3, cm: 'Up', i: 1, sn: 0 }); + assert.deepEqual(s.pack({ ids: [3, 1, 2] }), { is: [3, 1, 2] }); + // Left out, not sent as undefined. + assert.deepEqual(s.pack({ shadeId: 3, target: undefined }), { s: 3 }); + assert.throws(() => s.pack({ nonsense: 1 }), /no field nonsense/); +}); + +test('a field left out is its default', () => { + const s = recordedSchema(); + assert.equal(s.defaultOf('myPos'), -1); + assert.equal(s.defaultOf('stepSize'), 100); + assert.equal(s.defaultOf('bitLength'), 56); + assert.equal(s.defaultOf('paired'), false); + assert.equal(s.defaultOf('name'), ''); + assert.deepEqual(s.defaultOf('linkedShades'), []); + assert.deepEqual(s.fill({ shadeId: 2 }, ['shadeId', 'position', 'myPos']), { shadeId: 2, position: 0, myPos: -1 }); +}); + +test('a cached schema comes back whole, with its hash', () => { + const s = recordedSchema(); + const back = ProtoSchema.load(JSON.parse(JSON.stringify(s.save()))); + assert.equal(back.hash, s.hash); + assert.deepEqual(back.fields, s.fields); + assert.deepEqual(back.types, s.types); + assert.equal(ProtoSchema.load(null), null); + assert.equal(ProtoSchema.load({ hash: 'x', rows: {} }), null); +}); From 4415beb4f72cc5dcef6d02ccadded6e1e0b1c369 Mon Sep 17 00:00:00 2001 From: cjkas Date: Sun, 25 Oct 2026 18:04:07 +0100 Subject: [PATCH 2/7] fix: web UI client review findings (step 2a) - One socket at a time: a socket being replaced is let go without a word, and connect() or a new token while a reconnect waits opens no second one. A new token reconnects at once, without a refused. - Events that came before the schema go as soon as it is whole, ahead of the sync frames formatted after them, not at syncEnd over newer state. - A schema's hash counts only once it has come whole, so a sync cut short gets it again. - Feeds are subscribed again after a reconnect; an unknown feed is refused. - A long op refused as busy without its id goes again. - A sync without repeaters clears them; order changes name their list; a refused resync is called off. - refused: a listener can close() to stop the retries, which the UI does once /loginContext says a login is needed. The client itself can't tell a refused token from a device that is down. - localStorage keeps the schema by default. minify.py removes its work folder even when a bundle fails. ADR 0005 records the two bundles. - Smaller: ProtoSchema.ROWS/withoutRows, ProtoModel.ITEMS, named type numbers, unused feeds map and FakeSocket.all gone. Co-Authored-By: Claude Opus 5.5 --- data-src/js/client.js | 202 +++++++++++++++++------- data-src/js/model.js | 107 +++++++------ data-src/js/schema.js | 17 +- docs/adr/0005-one-websocket-protocol.md | 1 + minify.py | 26 +-- test/js/client.test.js | 131 +++++++++++++-- test/js/helpers.js | 24 +-- test/js/model.test.js | 21 ++- 8 files changed, 382 insertions(+), 147 deletions(-) diff --git a/data-src/js/client.js b/data-src/js/client.js index 9adb85b..fff68a3 100644 --- a/data-src/js/client.js +++ b/data-src/js/client.js @@ -15,15 +15,15 @@ class ProtoError extends Error { } } class ProtoClient { - // opts: host, token, WebSocket, storage (getItem/setItem, for the schema), - // setTimeout/clearTimeout, opTimeoutMs. + // opts: host, token, WebSocket, storage (getItem/setItem for the schema; + // localStorage when there is one), setTimeout/clearTimeout, opTimeoutMs. constructor(opts) { const deps = ProtoClient.deps(); this.Schema = deps.ProtoSchema; this.host = opts.host; this.token = opts.token || ''; this.WebSocket = opts.WebSocket || (typeof WebSocket !== 'undefined' ? WebSocket : null); - this.storage = opts.storage || null; + this.storage = typeof opts.storage !== 'undefined' ? opts.storage : ProtoClient.browserStorage(); this.setTimeout = opts.setTimeout || ((fn, ms) => setTimeout(fn, ms)); this.clearTimeout = opts.clearTimeout || ((t) => clearTimeout(t)); this.opTimeoutMs = opts.opTimeoutMs || 10000; @@ -32,9 +32,12 @@ class ProtoClient { this.model.onChange((kind, id, item) => this.emit('change', { kind: kind, id: id, item: item })); this.listeners = {}; this.pending = new Map(); + this.feeds = new Set(); this.nextId = 1; this.ws = null; - this.ready = false; + this.ready = false; // synced: ops may go + this.schemaReady = false; // frames can be read + this.helloHash = null; this.opened = false; this.stopped = true; this.retries = 0; @@ -49,29 +52,41 @@ class ProtoClient { return { ProtoSchema: require('./schema.js').ProtoSchema, ProtoModel: require('./model.js').ProtoModel }; return { ProtoSchema: ProtoSchema, ProtoModel: ProtoModel }; } + static browserStorage() { + try { + return typeof localStorage !== 'undefined' ? localStorage : null; + } catch (err) { + return null; + } + } // cb(d) for a device event by its name (shadeState, frame ...), and for - // the client's own: open, hello, synced, change, refused (closed before it - // opened: the token may have been refused), closed, versionMismatch and - // error (a refusal the device couldn't tie to an op). + // the client's own: open, hello, synced, change, closed, versionMismatch, + // error (a refusal the device couldn't tie to an op) and refused. + // refused: the socket closed before it opened. A browser can't tell a + // refused token from a device that is down, so the client tries again + // unless a listener calls close(), as the UI does once /loginContext says + // a login is needed (docs/SECURITY_PLAN.md). on(name, cb) { (this.listeners[name] = this.listeners[name] || []).push(cb); } emit(name, d) { for (const cb of this.listeners[name] || []) cb(d); } connect() { + if (!this.stopped && (this.ws || this.reconnectTimer)) return; this.stopped = false; this.open(); } close() { this.stopped = true; - if (this.reconnectTimer) this.clearTimeout(this.reconnectTimer); - this.reconnectTimer = null; + this.cancelReconnect(); if (this.ws) this.ws.close(); } - // A new token, from a login: the device checks it as the socket opens. + // A new token, from a login: the device checks it as the socket opens, so + // the client connects again with it at once. setToken(token) { this.token = token || ''; - if (this.ws) this.ws.close(); - else if (!this.stopped) this.open(); + this.stopped = false; + this.retries = 0; + this.open(); } url() { const params = []; @@ -79,28 +94,38 @@ class ProtoClient { if (this.schema.hash) params.push(`schema=${encodeURIComponent(this.schema.hash)}`); return `ws://${this.host}/ws${params.length ? '?' + params.join('&') : ''}`; } - open() { + cancelReconnect() { + if (this.reconnectTimer) this.clearTimeout(this.reconnectTimer); this.reconnectTimer = null; + } + // One socket at a time: one being replaced is let go without a word. + open() { + this.cancelReconnect(); + if (this.ws) { + const old = this.ws; + this.ws = null; + this.lost(); + old.close(); + } this.ready = false; + this.schemaReady = false; this.opened = false; this.held = []; const ws = new this.WebSocket(this.url()); this.ws = ws; ws.onopen = () => { + if (this.ws !== ws) return; this.opened = true; this.retries = 0; this.emit('open'); }; - ws.onmessage = (evt) => this.receive(evt.data); + ws.onmessage = (evt) => { + if (this.ws === ws) this.receive(evt.data); + }; ws.onclose = () => { if (this.ws !== ws) return; this.ws = null; - this.ready = false; - for (const [, p] of this.pending) { - this.clearTimeout(p.timer); - p.reject(new ProtoError('closed', 'The connection closed')); - } - this.pending.clear(); + this.lost(); this.emit(this.opened ? 'closed' : 'refused'); if (this.stopped) return; // Soon after a close, slower after each failure, at most 10 s. @@ -108,12 +133,21 @@ class ProtoClient { this.reconnectTimer = this.setTimeout(() => this.open(), ms); }; } - send(type, d, id) { + // The socket has gone: nothing waiting gets its reply. + lost() { + this.ready = false; + for (const [, p] of this.pending) { + this.clearTimeout(p.timer); + p.reject(new ProtoError('closed', 'The connection closed')); + } + this.pending.clear(); + } + frame(type, d, id) { const frame = { v: ProtoClient.VERSION, t: this.schema.typeNumbers[type] }; if (typeof frame.t === 'undefined') throw new ProtoError('unknownOp', `The device has no message ${type}`); if (id) frame.id = id; if (d) frame.d = d; - this.ws.send(JSON.stringify(frame)); + return JSON.stringify(frame); } // Sends an op, or a control message that wants a reply, by its name with @@ -126,25 +160,39 @@ class ProtoClient { return; } const id = this.nextId++; + let text; try { - this.send(name, this.schema.pack(fields || {}), id); + text = this.frame(name, this.schema.pack(fields || {}), id); } catch (err) { reject(err instanceof ProtoError ? err : new ProtoError('badRequest', err.message)); return; } + this.ws.send(text); const timer = this.setTimeout(() => { this.pending.delete(id); reject(new ProtoError('timeout', `No reply to ${name}`)); }, this.opTimeoutMs); - this.pending.set(id, { resolve: resolve, reject: reject, timer: timer }); + this.pending.set(id, { resolve: resolve, reject: reject, timer: timer, text: text, resends: 0 }); }); } - subscribe(feed) { return this.op('subscribe', { feed: this.schema.feedNumbers[feed] }); } - unsubscribe(feed) { return this.op('unsubscribe', { feed: this.schema.feedNumbers[feed] }); } + // Feeds stay subscribed across reconnects. + subscribe(feed) { + if (typeof this.schema.feedNumbers[feed] === 'undefined') return Promise.reject(new ProtoError('badRequest', `The device has no feed ${feed}`)); + this.feeds.add(feed); + return this.op('subscribe', { feed: this.schema.feedNumbers[feed] }); + } + unsubscribe(feed) { + this.feeds.delete(feed); + if (typeof this.schema.feedNumbers[feed] === 'undefined') return Promise.resolve(); + return this.op('unsubscribe', { feed: this.schema.feedNumbers[feed] }); + } // The shade model and status again, without the schema. resync() { this.model.beginSync(); - return this.op('sync'); + return this.op('sync').catch((err) => { + this.model.cancelSync(); + throw err; + }); } receive(text) { @@ -159,67 +207,99 @@ class ProtoClient { return; } const d = frame.d || {}; - // hello, the sync and syncEnd have fixed numbers: they come before the - // schema. + // These have fixed numbers: they come before the schema. switch (frame.t) { - case 1: this.hello(d); return; - case 4: this.syncFrame(d); return; - case 5: this.syncEnd(); return; + case ProtoClient.HELLO: this.hello(d); return; + case ProtoClient.SYNC: this.syncFrame(d); return; + case ProtoClient.SYNC_END: this.syncEnd(); return; default: break; } - if (!this.ready) this.held.push(frame); + // A frame the client can't read yet waits for the schema. + if (!this.schemaReady) this.held.push(frame); else this.dispatch(frame); } hello(d) { this.fw = d.fw; - // A schema cached for another firmware is no use: the device sends the - // one it has, as it didn't get its hash. + this.helloHash = d.sh; + // A schema the client kept for this one needn't come again; any other + // is no use, and the device sends the one it has. if (this.schema.hash !== d.sh) { this.schema = new this.Schema(); this.model.schema = this.schema; } - this.schema.hash = d.sh; this.model.beginSync(); this.emit('hello', d); + if (this.schema.hash === d.sh) this.readable(); } syncFrame(d) { this.schema.add(d); - const device = {}; - for (const key of Object.keys(d)) if (!['T', 'F', 'E', 'Q'].includes(key)) device[key] = d[key]; + const device = this.Schema.withoutRows(d); // The schema's rows come first, so a frame with the device's sections // has the whole of it. - if (Object.keys(device).length) this.model.applySync(this.schema.expand(device)); - this.ws.send(JSON.stringify({ v: ProtoClient.VERSION, t: 3 })); + if (Object.keys(device).length) { + this.readable(); + this.model.applySync(this.schema.expand(device)); + } + this.ws.send(JSON.stringify({ v: ProtoClient.VERSION, t: ProtoClient.NEXT })); + } + // The schema is whole: frames that waited for it go, in the order they + // came and ahead of the sync frames formatted after them. + readable() { + if (this.schemaReady) return; + this.schemaReady = true; + const held = this.held; + this.held = []; + for (const frame of held) this.dispatch(frame); } syncEnd() { + this.readable(); this.model.endSync(); if (!this.ready) { this.ready = true; + // Only a whole schema is kept, so a sync cut short gets it again. + this.schema.hash = this.helloHash; this.saveSchema(); - const held = this.held; - this.held = []; - for (const frame of held) this.dispatch(frame); + for (const feed of this.feeds) this.op('subscribe', { feed: this.schema.feedNumbers[feed] }).catch(() => {}); } this.emit('synced', this.model); } dispatch(frame) { const d = this.schema.expand(frame.d || {}); - if (frame.t === 2) { - const p = typeof frame.id !== 'undefined' ? this.pending.get(frame.id) : null; - if (!p) { - if (!d.ok) this.emit('error', new ProtoError(this.schema.errors[d.code] || 'internal', d.desc, d)); - return; - } - this.pending.delete(frame.id); + if (frame.t === ProtoClient.REPLY) this.reply(frame.id, d); + else { + const name = this.schema.types[frame.t]; + if (!name) return; + this.model.applyEvent(name, d); + this.emit(name, d); + } + } + reply(id, d) { + const err = d.ok ? null : new ProtoError(this.schema.errors[d.code] || 'internal', d.desc, d); + const p = typeof id !== 'undefined' ? this.pending.get(id) : null; + if (p) { + this.pending.delete(id); this.clearTimeout(p.timer); - if (d.ok) p.resolve(d); - else p.reject(new ProtoError(this.schema.errors[d.code] || 'internal', d.desc, d)); + if (err) p.reject(err); + else p.resolve(d); return; } - const name = this.schema.types[frame.t]; - if (!name) return; - this.model.applyEvent(name, d); - this.emit(name, d); + if (!err) return; + // A long message the device couldn't take while it put another one + // together is refused without its id: the long ones waiting go again. + if (err.code === 'busy' && this.resendLong()) return; + this.emit('error', err); + } + resendLong() { + let any = false; + for (const [, p] of this.pending) { + if (p.text.length < ProtoClient.LONG_MESSAGE || p.resends >= 3) continue; + p.resends++; + any = true; + this.setTimeout(() => { + if (this.ws && [...this.pending.values()].includes(p)) this.ws.send(p.text); + }, 200 * p.resends); + } + return any; } loadSchema() { @@ -238,5 +318,15 @@ class ProtoClient { } } ProtoClient.VERSION = 2; +// The types a client reads before it has the schema, and next, which it sends +// during the sync. +ProtoClient.HELLO = 1; +ProtoClient.REPLY = 2; +ProtoClient.NEXT = 3; +ProtoClient.SYNC = 4; +ProtoClient.SYNC_END = 5; +// A message longer than a TCP segment reaches the device in pieces +// (docs/PROTOCOL.md). +ProtoClient.LONG_MESSAGE = 1400; ProtoClient.SCHEMA_KEY = 'espsomfyProtoSchema'; if (typeof module !== 'undefined' && module.exports) module.exports = { ProtoClient, ProtoError }; diff --git a/data-src/js/model.js b/data-src/js/model.js index 2ab15e4..2411996 100644 --- a/data-src/js/model.js +++ b/data-src/js/model.js @@ -17,21 +17,24 @@ class ProtoModel { this.listeners = []; this.seen = null; } - // cb(kind, id, item): kind is room, shade, group, roomRemoved, - // shadeRemoved, groupRemoved, order, repeaters, header or status. + // cb(kind, id, item): kind is room, shade or group for an item added or + // changed; roomRemoved, shadeRemoved or groupRemoved; roomOrder, + // shadeOrder or groupOrder (item: the ids in order); repeaters, header or + // status. onChange(cb) { this.listeners.push(cb); } changed(kind, id, item) { for (const cb of this.listeners) cb(kind, id, item); } - // A sync: the items it doesn't list are gone once it ends. - beginSync() { this.seen = { rooms: new Set(), shades: new Set(), groups: new Set() }; } + // A sync: what it doesn't list is gone once it ends, the repeaters too, + // which it leaves out when there are none. + beginSync() { this.seen = { rooms: new Set(), shades: new Set(), groups: new Set(), repeaters: false }; } + // A sync that won't come after all. + cancelSync() { this.seen = null; } applySync(d) { if (d.header) { this.header = this.schema.fill(Object.assign({}, d.header), ProtoModel.FIELDS.header); this.changed('header', 0, this.header); } - for (const r of d.rooms || []) this.putRoom(r); - for (const s of d.shades || []) this.putShade(s); - for (const g of d.groups || []) this.putGroup(g); + for (const kind of ProtoModel.ITEMS) for (const item of d[kind.list] || []) this.put(kind, item); if (d.repeaters) this.putRepeaters(d.repeaters); if (d.status) { this.putStatus(d.status, ProtoModel.FIELDS.memory); @@ -40,30 +43,31 @@ class ProtoModel { } endSync() { if (!this.seen) return; - const lists = [['rooms', 'roomRemoved'], ['shades', 'shadeRemoved'], ['groups', 'groupRemoved']]; - for (const [list, removed] of lists) { - for (const id of [...this[list].keys()]) { - if (this.seen[list].has(id)) continue; - this[list].delete(id); - this.changed(removed, id); + for (const kind of ProtoModel.ITEMS) { + const map = this[kind.list]; + for (const id of [...map.keys()]) { + if (this.seen[kind.list].has(id)) continue; + map.delete(id); + this.changed(kind.removed, id); } } + if (!this.seen.repeaters && this.repeaters.length) this.putRepeaters([]); this.seen = null; } - // An event by its type's name; false for one that isn't about the model. + // An event by its type's name; false for one that isn't about the model, + // or that changed nothing. applyEvent(type, d) { + for (const kind of ProtoModel.ITEMS) { + const id = d[kind.id] || 0; + switch (type) { + case kind.name: this.put(kind, d); return true; + case kind.state: return this.mergeState(kind, id, d); + case kind.removed: return this.remove(kind, id); + case kind.order: return this.order(kind, d.ids || []); + default: break; + } + } switch (type) { - case 'room': this.putRoom(d); return true; - case 'shade': this.putShade(d); return true; - case 'group': this.putGroup(d); return true; - case 'shadeState': return this.mergeState(this.shades, 'shade', d.shadeId, d, ProtoModel.FIELDS.shadeState); - case 'groupState': return this.mergeState(this.groups, 'group', d.groupId, d, ProtoModel.FIELDS.groupState); - case 'roomRemoved': return this.remove(this.rooms, type, d.roomId || 0); - case 'shadeRemoved': return this.remove(this.shades, type, d.shadeId || 0); - case 'groupRemoved': return this.remove(this.groups, type, d.groupId || 0); - case 'roomOrder': return this.order(this.rooms, d.ids || []); - case 'shadeOrder': return this.order(this.shades, d.ids || []); - case 'groupOrder': return this.order(this.groups, d.ids || []); case 'repeaters': this.putRepeaters(d.repeaters || []); return true; case 'memory': this.putStatus(d, ProtoModel.FIELDS.memory); return true; case 'wifi': this.putStatus(d, ProtoModel.FIELDS.wifi); return true; @@ -78,45 +82,42 @@ class ProtoModel { groupList() { return ProtoModel.sorted(this.groups); } static sorted(map) { return [...map.values()].sort((a, b) => a.sortOrder - b.sortOrder); } - putRoom(r) { this.put(this.rooms, 'room', r.roomId, r, ProtoModel.FIELDS.room, 'rooms'); } - putShade(s) { - // Its linked remotes leave out their defaults too. - const linked = (s.linkedRemotes || []).map((r) => this.schema.fill(Object.assign({}, r), ProtoModel.FIELDS.linkedRemote)); - this.put(this.shades, 'shade', s.shadeId, Object.assign({}, s, { linkedRemotes: linked }), ProtoModel.FIELDS.shade, 'shades'); - } - putGroup(g) { this.put(this.groups, 'group', g.groupId, g, ProtoModel.FIELDS.group, 'groups'); } // A whole item: what it had before is replaced. - put(map, kind, id, item, fields, list) { - id = id || 0; - const whole = this.schema.fill(Object.assign({}, item), fields); - map.set(id, whole); - if (this.seen) this.seen[list].add(id); - this.changed(kind, id, whole); + put(kind, item) { + const id = item[kind.id] || 0; + const whole = this.schema.fill(Object.assign({}, item), ProtoModel.FIELDS[kind.name]); + // A shade's linked remotes leave out their defaults too. + if (kind.name === 'shade') + whole.linkedRemotes = whole.linkedRemotes.map((r) => this.schema.fill(Object.assign({}, r), ProtoModel.FIELDS.linkedRemote)); + this[kind.list].set(id, whole); + if (this.seen) this.seen[kind.list].add(id); + this.changed(kind.name, id, whole); } // Part of an item: only its state. A state of an item not synced yet // waits for the item. - mergeState(map, kind, id, d, fields) { - const item = map.get(id || 0); + mergeState(kind, id, d) { + const item = this[kind.list].get(id); if (!item) return false; - Object.assign(item, this.schema.fill(Object.assign({}, d), fields)); - this.changed(kind, item[`${kind}Id`], item); + Object.assign(item, this.schema.fill(Object.assign({}, d), ProtoModel.FIELDS[kind.state])); + this.changed(kind.name, id, item); return true; } - remove(map, kind, id) { - if (!map.delete(id)) return false; - this.changed(kind, id); + remove(kind, id) { + if (!this[kind.list].delete(id)) return false; + this.changed(kind.removed, id); return true; } - order(map, ids) { + order(kind, ids) { ids.forEach((id, i) => { - const item = map.get(id); + const item = this[kind.list].get(id); if (item) item.sortOrder = i; }); - this.changed('order', 0, ids); + this.changed(kind.order, 0, ids); return true; } putRepeaters(list) { this.repeaters = list.slice(); + if (this.seen) this.seen.repeaters = true; this.changed('repeaters', 0, this.repeaters); } putStatus(d, fields) { @@ -126,9 +127,17 @@ class ProtoModel { this.changed('status', 0, this.status); } } +// Rooms, shades and groups: their list, id, and the events about them. Rooms +// have no state of their own. +ProtoModel.ITEMS = [ + { name: 'room', list: 'rooms', id: 'roomId', state: null, removed: 'roomRemoved', order: 'roomOrder' }, + { name: 'shade', list: 'shades', id: 'shadeId', state: 'shadeState', removed: 'shadeRemoved', order: 'shadeOrder' }, + { name: 'group', list: 'groups', id: 'groupId', state: 'groupState', removed: 'groupRemoved', order: 'groupOrder' } +]; // The fields of each kind of item, as the device writes them // (src/ProtoSync.cpp, src/outputs/ProtoSocket.cpp): what a frame that leaves -// one out means by it. +// one out means by it. The schema has each field's default, not which items +// have it. ProtoModel.FIELDS = { header: ['maxRooms', 'maxShades', 'maxGroups', 'maxGroupedShades', 'maxLinkedRemotes', 'startingAddress', 'bitLength', 'proto', 'serverId', 'hostname', 'model', 'chipModel', 'fw', 'connType'], diff --git a/data-src/js/schema.js b/data-src/js/schema.js index 50e34ea..f0cf0f2 100644 --- a/data-src/js/schema.js +++ b/data-src/js/schema.js @@ -11,10 +11,15 @@ class ProtoSchema { this.keys = {}; // name -> key this.errors = {}; this.errorNumbers = {}; - this.feeds = {}; this.feedNumbers = {}; this.rows = { T: [], F: [], E: [], Q: [] }; } + // A sync frame's payload without the schema's rows: the device's sections. + static withoutRows(d) { + const out = {}; + for (const key of Object.keys(d)) if (!ProtoSchema.ROWS.includes(key)) out[key] = d[key]; + return out; + } // The schema's rows in a sync frame's payload: T types, F fields, E error // codes, Q feeds. They may span several frames. add(d) { @@ -30,11 +35,8 @@ class ProtoSchema { this.errors[number] = name; this.errorNumbers[name] = number; } - for (const [number, name] of d.Q || []) { - this.feeds[number] = name; - this.feedNumbers[name] = number; - } - for (const k of ['T', 'F', 'E', 'Q']) if (d[k]) this.rows[k].push(...d[k]); + for (const [number, name] of d.Q || []) this.feedNumbers[name] = number; + for (const k of ProtoSchema.ROWS) if (d[k]) this.rows[k].push(...d[k]); } get empty() { return Object.keys(this.fields).length === 0; } // What a field left out of a frame is: its default. @@ -93,4 +95,7 @@ class ProtoSchema { return s.empty ? null : s; } } +// The keys of the schema's rows in a sync frame, fixed as a client needs them +// before it has the schema. +ProtoSchema.ROWS = ['T', 'F', 'E', 'Q']; if (typeof module !== 'undefined' && module.exports) module.exports = { ProtoSchema }; diff --git a/docs/adr/0005-one-websocket-protocol.md b/docs/adr/0005-one-websocket-protocol.md index 9f862fd..8d008fb 100644 --- a/docs/adr/0005-one-websocket-protocol.md +++ b/docs/adr/0005-one-websocket-protocol.md @@ -22,6 +22,7 @@ We decided that the web UI and the Home Assistant integration do everything exce - **The send queue stays at 32 messages per client.** A full queue closes the client, which reconnects and syncs again. Dropping messages would leave a shade showing the wrong position with no way to notice. - **A JSON envelope that a binary encoding can reuse.** Every frame is `{"v":2,"t":,"id":,"d":{...}}`, with `id` only on ops and replies. Message types and error codes are numbers that fit one byte and are never reused. A later binary encoding would put the same four fields in a fixed header, be asked for at the upgrade, and use binary websocket frames. The `42[...]` socket.io framing goes. - **Short keys, described by a schema the device sends.** Payload keys are one or two characters from one global dictionary. Booleans are 1/0, and fields at their default are left out. The schema lists every type, field (name, kind, default) and error code. The firmware's table is the only copy, and its writers can only emit declared fields. `hello` carries the schema's hash, and a client that has it cached reconnects with `?schema=` to skip it. Clients expand frames to full names, so client code stays readable. +- **The web UI loads two files of its own: `index.js` and `app.css`.** Each file a page load fetches costs the device an open response, so the minify step joins the CSS files into `app.css`, and the protocol 2 client (`data-src/js/`) into `index.js`. Each client module ends with a `module.exports` footer, so Node can load it for the tests, and in the browser it adds nothing global but its classes. - **Feeds.** High-rate events (received frames, frequency scan) go only to clients that subscribed to that feed. - **A clean break, released as 4.0.0.** The firmware speaks only protocol 2. The new HA integration falls back to the old HTTP API and events when a device doesn't send `hello`; old integrations stop working against 4.0.0. The web UI checks `v` and shows a banner when the firmware and the web application don't match. - **Tests.** Native tests cover parsing, the op table, permissions, the sync stream and the largest item fitting in 4 KB. A Python e2e suite, run by hand against a device, checks the protocol and survival under a reconnect storm and a page-load replay, and records fixtures. The web UI's client layer is tested with `node --test`. The HA integration tests its protocol module with plain pytest and the integration with pytest-homeassistant-custom-component, against a fake device that replays the fixtures. diff --git a/minify.py b/minify.py index f7b48d6..b92de59 100644 --- a/minify.py +++ b/minify.py @@ -275,19 +275,23 @@ def minify_all(): file_count += 1 bundled = set() + # Inside data/, so it goes even when a bundle fails: left there, it would + # end up in the LittleFS image. work_dir = os.path.join(dst_dir, ".bundles") os.makedirs(work_dir, exist_ok=True) - for name, parts in BUNDLES.items(): - joined = os.path.join(work_dir, name) - with open(joined, "w", encoding="utf-8") as out: - for part in parts: - with open(os.path.join(src_dir, part), "r", encoding="utf-8") as f: - out.write(f.read()) - # A file without a final newline would run into the next. - out.write("\n") - bundled.add(os.path.normpath(part)) - report(f"{name} ({len(parts)} files)", process_file(joined, os.path.join(dst_dir, name))) - shutil.rmtree(work_dir) + try: + for name, parts in BUNDLES.items(): + joined = os.path.join(work_dir, name) + with open(joined, "w", encoding="utf-8") as out: + for part in parts: + with open(os.path.join(src_dir, part), "r", encoding="utf-8") as f: + out.write(f.read()) + # A file without a final newline would run into the next. + out.write("\n") + bundled.add(os.path.normpath(part)) + report(f"{name} ({len(parts)} files)", process_file(joined, os.path.join(dst_dir, name))) + finally: + shutil.rmtree(work_dir, ignore_errors=True) for root, dirs, files in os.walk(src_dir): for fname in sorted(files): diff --git a/test/js/client.test.js b/test/js/client.test.js index 17d5c39..3966007 100644 --- a/test/js/client.test.js +++ b/test/js/client.test.js @@ -3,13 +3,13 @@ const test = require('node:test'); const assert = require('node:assert/strict'); const { ProtoClient, ProtoError } = require('../../data-src/js/client.js'); -const { hello, sync, recordedItems, recordedSchema, FakeSocket, FakeTimers, FakeStorage } = require('./helpers.js'); +const { hello, sync, syncWithoutSchema, recordedItems, recordedSchema, FakeSocket, FakeTimers, FakeStorage } = require('./helpers.js'); function client(opts) { FakeSocket.reset(); const timers = new FakeTimers(); const c = new ProtoClient(Object.assign({ host: 'device', WebSocket: FakeSocket, setTimeout: timers.setTimeout, - clearTimeout: timers.clearTimeout }, opts || {})); + clearTimeout: timers.clearTimeout, storage: null }, opts || {})); c.timers = timers; return c; } @@ -131,17 +131,36 @@ test('events update the model, and are told by name', () => { assert.deepEqual(seen, [{ shadeId: shade.shadeId, target: 30, direction: 1 }]); }); -test('an event before the schema is kept until the sync ends', () => { +test('an event before the schema waits for it, and goes before the sync frames formatted after it', () => { const c = client(); + const states = []; + c.on('shadeState', (d) => states.push(d)); c.connect(); const ws = FakeSocket.last; ws.open(); ws.deliver(hello); // Before the schema: the client can't read it yet. - const shadeId = recordedItems('S')[0].s; - ws.deliver({ v: 2, t: T.shadeState, d: { s: shadeId, t: 55 } }); + const recorded = recordedItems('S')[0]; + ws.deliver({ v: 2, t: T.shadeState, d: { s: recorded.s, t: 55 } }); + assert.deepEqual(states, []); for (const f of sync) ws.deliver(f); - assert.equal(c.model.shades.get(shadeId).target, 55); + assert.deepEqual(states, [{ shadeId: recorded.s, target: 55 }]); + // The shade's sync frame was formatted after the event, so it wins. + assert.equal(c.model.shades.get(recorded.s).target, recorded.t || 0); +}); + +test('an event after the schema goes at once, during the sync', () => { + const c = client(); + c.connect(); + const ws = FakeSocket.last; + ws.open(); + ws.deliver(hello); + const syncFrames = sync.filter((f) => f.t === T.sync); + for (const f of syncFrames) ws.deliver(f); + const shadeId = recordedItems('S')[0].s; + ws.deliver({ v: 2, t: T.shadeState, d: { s: shadeId, t: 66 } }); + assert.equal(c.ready, false); + assert.equal(c.model.shades.get(shadeId).target, 66); }); test('the schema is kept, and the device is told it has it', () => { @@ -153,16 +172,101 @@ test('the schema is kept, and the device is told it has it', () => { assert.equal(ws.url, `ws://device/ws?schema=${hello.d.sh}`); ws.open(); ws.deliver(hello); - // Without the schema's rows, as the device sends a client that has it. - for (const f of sync) { - const d = Object.assign({}, f.d); - for (const k of ['T', 'F', 'E', 'Q']) delete d[k]; - ws.deliver(Object.assign({}, f, { d: d })); - } + for (const f of syncWithoutSchema()) ws.deliver(f); assert.ok(c.ready); assert.equal(c.model.shades.size, recordedItems('S').length); }); +test('a schema cut off by a lost socket is not kept, so it comes whole again', () => { + const c = client(); + c.connect(); + const ws = FakeSocket.last; + ws.open(); + ws.deliver(hello); + ws.deliver(sync[0]); + ws.drop(); + c.timers.runAll(); + assert.equal(FakeSocket.last.url, 'ws://device/ws'); + FakeSocket.last.open(); + FakeSocket.last.deliver(hello); + for (const f of sync) FakeSocket.last.deliver(f); + assert.equal(Object.keys(c.schema.fields).length, Object.keys(schema.fields).length); + assert.equal(c.schema.hash, hello.d.sh); +}); + +test('a socket is replaced, never doubled', () => { + const { c, ws } = connected(); + c.connect(); + assert.equal(FakeSocket.count, 1); + // A new token while a reconnect is waiting: one socket, and no timer left. + ws.drop(); + c.setToken('new'); + assert.equal(FakeSocket.count, 2); + assert.deepEqual(c.timers.runAll(), []); + // While it connects: the old one goes without a refused. + let refused = 0; + c.on('refused', () => refused++); + c.setToken('newer'); + assert.equal(refused, 0); + assert.equal(FakeSocket.count, 3); + assert.match(FakeSocket.last.url, /apikey=newer/); +}); + +test('a listener can stop the retries after a refusal', () => { + const c = client(); + c.on('refused', () => c.close()); + c.connect(); + FakeSocket.last.drop(); + assert.deepEqual(c.timers.runAll(), []); + assert.equal(FakeSocket.count, 1); +}); + +test('feeds are subscribed again after a reconnect', () => { + const { c, ws } = connected(); + // Its reply never comes: the socket closes first. + c.subscribe('frames').catch(() => {}); + ws.drop(); + c.timers.runAll(); + const again = FakeSocket.last; + again.open(); + again.deliver(hello); + for (const f of sync) again.deliver(f); + const sent = again.sent.filter((f) => f.t === T.subscribe); + assert.deepEqual(sent.map((f) => f.d), [{ fe: schema.feedNumbers.frames }]); +}); + +test('a feed the device lacks is refused here', async () => { + const { c } = connected(); + await assert.rejects(c.subscribe('nope'), (err) => err.code === 'badRequest'); +}); + +test('a long op refused as busy without its id goes again', () => { + const { c, ws } = connected(); + c.op('saveRoom', { roomId: 1, name: 'x'.repeat(1500) }).catch(() => {}); + const sent = ws.sent.length; + ws.deliver(reply(undefined, { e: schema.errorNumbers.busy, x: 'Another long message is on its way' })); + c.timers.runAll(); + assert.equal(ws.sent.length, sent + 1); + assert.equal(ws.sent[ws.sent.length - 1].d.n.length, 1500); +}); + +test('a refusal without an id for nothing long is reported', () => { + const { c, ws } = connected(); + let error = null; + c.on('error', (err) => { error = err; }); + ws.deliver(reply(undefined, { e: schema.errorNumbers.badRequest, x: 'Malformed message' })); + assert.equal(error.code, 'badRequest'); +}); + +test('a sync asked for and refused is called off', async () => { + const { c, ws } = connected(); + const p = c.resync(); + const id = ws.sent[ws.sent.length - 1].id; + ws.deliver(reply(id, { e: schema.errorNumbers.busy, x: 'busy' })); + await assert.rejects(p); + assert.equal(c.model.seen, null); +}); + test('a kept schema of another firmware is dropped', () => { const storage = new FakeStorage(); connected({ storage: storage }); @@ -171,7 +275,8 @@ test('a kept schema of another firmware is dropped', () => { const ws = FakeSocket.last; ws.open(); ws.deliver({ v: 2, t: 1, d: { fw: 'v4.0.0', sh: 'ffffffff' } }); - assert.equal(c.schema.hash, 'ffffffff'); + // Its hash only counts once the new one has come whole. + assert.equal(c.schema.hash, null); assert.ok(c.schema.empty); }); diff --git a/test/js/helpers.js b/test/js/helpers.js index fd73718..e452575 100644 --- a/test/js/helpers.js +++ b/test/js/helpers.js @@ -17,11 +17,11 @@ function recordedSchema() { } // The recorded sync's device sections, expanded, frame by frame. function deviceSections(schema) { - return sync.filter((f) => f.t === 4).map((f) => { - const d = {}; - for (const k of Object.keys(f.d)) if (!['T', 'F', 'E', 'Q'].includes(k)) d[k] = f.d[k]; - return schema.expand(d); - }); + return sync.filter((f) => f.t === 4).map((f) => schema.expand(ProtoSchema.withoutRows(f.d))); +} +// The recorded sync as the device sends it to a client that has the schema. +function syncWithoutSchema() { + return sync.map((f) => Object.assign({}, f, { d: ProtoSchema.withoutRows(f.d || {}) })); } // The recorded sync's items under their short keys. function recordedItems(key) { @@ -36,7 +36,7 @@ class FakeSocket { this.sent = []; this.closed = false; FakeSocket.last = this; - FakeSocket.all.push(this); + FakeSocket.count++; } send(text) { this.sent.push(JSON.parse(text)); } close() { @@ -49,8 +49,9 @@ class FakeSocket { // The device closing it, or refusing it before it opened. drop() { this.close(); } } -FakeSocket.all = []; -FakeSocket.reset = () => { FakeSocket.all = []; FakeSocket.last = null; }; +// How many the client has opened. +FakeSocket.count = 0; +FakeSocket.reset = () => { FakeSocket.count = 0; FakeSocket.last = null; }; // Timers the test runs by hand. class FakeTimers { @@ -64,9 +65,9 @@ class FakeTimers { }; this.clearTimeout = (id) => { this.timers.delete(id); }; } - // Runs every timer waiting, and returns their delays. + // Runs every timer waiting, the shortest first, and returns their delays. runAll() { - const due = [...this.timers.entries()]; + const due = [...this.timers.entries()].sort((a, b) => a[1].ms - b[1].ms); this.timers.clear(); for (const [, t] of due) t.fn(); return due.map(([, t]) => t.ms); @@ -79,4 +80,5 @@ class FakeStorage { setItem(key, value) { this.items[key] = String(value); } } -module.exports = { hello, sync, schemaJson, recordedSchema, deviceSections, recordedItems, FakeSocket, FakeTimers, FakeStorage }; +module.exports = { hello, sync, schemaJson, recordedSchema, deviceSections, syncWithoutSchema, recordedItems, FakeSocket, + FakeTimers, FakeStorage }; diff --git a/test/js/model.test.js b/test/js/model.test.js index a7dc239..3d8cd4e 100644 --- a/test/js/model.test.js +++ b/test/js/model.test.js @@ -79,7 +79,26 @@ test('removals, orders and repeaters', () => { assert.equal(m.shades.get(second.shadeId).sortOrder, 0); m.applyEvent('repeaters', { repeaters: [7, 8] }); assert.deepEqual(m.repeaters, [7, 8]); - assert.deepEqual(changes.map((c) => c[0]), ['shadeRemoved', 'order', 'repeaters']); + // An order says which list it is. + assert.deepEqual(changes.map((c) => c[0]), ['shadeRemoved', 'shadeOrder', 'repeaters']); +}); + +test('a sync without repeaters clears them, as the device leaves the list out when it has none', () => { + const m = synced(); + m.applyEvent('repeaters', { repeaters: [7] }); + m.beginSync(); + for (const d of deviceSections(m.schema)) m.applySync(Object.assign({}, d, { repeaters: undefined })); + m.endSync(); + assert.deepEqual(m.repeaters, []); +}); + +test('a sync that is called off removes nothing', () => { + const m = synced(); + const shades = m.shades.size; + m.beginSync(); + m.cancelSync(); + m.endSync(); + assert.equal(m.shades.size, shades); }); test('a sync drops what it no longer lists, and keeps what an event added during it', () => { From 0df740df6aebc2841b701a07b5a423af62477320 Mon Sep 17 00:00:00 2001 From: cjkas Date: Sun, 25 Oct 2026 20:12:54 +0100 Subject: [PATCH 3/7] feat: web UI on socket protocol 2 (step 2b) The UI talks to the device over the one socket at /ws: the sync fills the shade model, its events redraw the lists, and every read, command, edit, setting, scan and reboot is an op. Only /loginContext, /login and the file transfers stay on HTTP, so a page load opens no other connection. - The 8080 socket, editJSON/requestResult and the /controller, /rooms, /shades, /groups, /room, /shade, /group, settings and command routes are no longer used by the UI. - An edit's callback gets the item once its event is in (ProtoClient waitFor); busy edits and saves are sent again. - The token is kept in sessionStorage. When /ws is refused and /loginContext says a login is needed, the client stops retrying and the login panel opens. Backup and uploads send the token. - A banner shows when the device speaks another protocol version. - Removed dead code: the /pairShade call and its unused /unpairShade twin, loadLogin, login.html (broken, linked from nowhere), Secure Config Only's checkbox (protocol 2 has two modes) and the HTTP helpers nothing uses. Co-Authored-By: Claude Opus 5.5 --- data-src/index.html | 4 - data-src/index.js | 1407 +++++++++++++++------------------------- data-src/js/client.js | 22 +- data-src/login.html | 51 -- data-src/main.css | 11 + test/js/client.test.js | 19 + 6 files changed, 570 insertions(+), 944 deletions(-) delete mode 100644 data-src/login.html diff --git a/data-src/index.html b/data-src/index.html index 398f0f0..4772b99 100644 --- a/data-src/index.html +++ b/data-src/index.html @@ -179,10 +179,6 @@ -