From 99c6019431adcb638f7a70924525b830867087ef Mon Sep 17 00:00:00 2001 From: cjkas Date: Sun, 25 Oct 2026 15:00:23 +0100 Subject: [PATCH] feat: web UI protocol 2 client layer, JS and CSS bundles (step 2a) - data-src/js: ProtoSchema (schema from the sync, short keys and names, defaults, cached by hash), ProtoModel (rooms, shades, groups, repeaters, header and status from the sync and the events) and ProtoClient (/ws, hello, sync with next, op() as a Promise of its reply, events, feeds, reconnect). Nothing uses them yet: 2b moves the UI onto them. - minify.py joins js/*.js and index.js into index.js, and the three CSS files into app.css, so a page load opens fewer connections. - test/js: node --test on the three modules, replaying the recorded fixtures; a CI job runs it and the firmware build waits for it. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/build.yaml | 19 ++- AGENTS.md | 1 + data-src/index.html | 4 +- data-src/js/client.js | 242 +++++++++++++++++++++++++++++++++++ data-src/js/model.js | 150 ++++++++++++++++++++++ data-src/js/schema.js | 96 ++++++++++++++ data-src/login.html | 4 +- minify.py | 52 ++++++-- test/e2e/test_survival.py | 2 +- test/js/client.test.js | 200 +++++++++++++++++++++++++++++ test/js/helpers.js | 82 ++++++++++++ test/js/model.test.js | 106 +++++++++++++++ test/js/schema.test.js | 56 ++++++++ 13 files changed, 994 insertions(+), 20 deletions(-) create mode 100644 data-src/js/client.js create mode 100644 data-src/js/model.js create mode 100644 data-src/js/schema.js create mode 100644 test/js/client.test.js create mode 100644 test/js/helpers.js create mode 100644 test/js/model.test.js create mode 100644 test/js/schema.test.js diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index b94857f..562aa41 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -28,9 +28,26 @@ jobs: - name: Run native unit tests run: pio test -e native + webui: + name: Web UI tests + runs-on: ubuntu-latest + + steps: + - name: Check out code + uses: actions/checkout@v4 + + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: "22" + + # The protocol 2 client (data-src/js) against the recorded fixtures. + - name: Run web UI tests + run: node --test "test/js/*.test.js" + build: # Don't build or publish firmware that fails its tests. - needs: test + needs: [test, webui] permissions: write-all name: ${{ matrix.name }} runs-on: ubuntu-latest diff --git a/AGENTS.md b/AGENTS.md index c4089ab..55a0dbf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,6 +2,7 @@ - Fix any test or type errors until the whole suite is green. - Add or update tests for the code you change, even if nobody asked. - to test run `pio test -e native` +- the web UI's protocol 2 client (`data-src/js/`): `node --test "test/js/*.test.js"`. The minify step joins those files into `index.js`, so each one ends with a `module.exports` footer for Node and adds nothing global but its classes. ## Building - Firmware builds need pioarduino's Core (`pip install pioarduino`, or the pioarduino IDE extension). Stock PlatformIO Core refuses the pinned platform, see [ADR 0003](docs/adr/0003-pioarduino-idf55.md). diff --git a/data-src/index.html b/data-src/index.html index d52572d..398f0f0 100644 --- a/data-src/index.html +++ b/data-src/index.html @@ -8,9 +8,7 @@ - - - + diff --git a/data-src/js/client.js b/data-src/js/client.js new file mode 100644 index 0000000..9adb85b --- /dev/null +++ b/data-src/js/client.js @@ -0,0 +1,242 @@ +// A socket protocol 2 client (docs/PROTOCOL.md): ws://host/ws, hello, the sync +// with next after each frame, ops answered by their reply, events and feeds. +// It keeps the shade model (ProtoModel) up to date and connects again when the +// socket closes. The WebSocket, the timers and the schema's storage can be +// handed in, so the tests (test/js) run it without a device. The minify step +// joins this file into index.js after schema.js and model.js. +class ProtoError extends Error { + // code is an error name from the schema (badRequest, busy ...), or + // notConnected, closed or timeout from the client itself. + constructor(code, desc, data) { + super(desc || code); + this.code = code; + this.desc = desc || code; + this.data = data || {}; + } +} +class ProtoClient { + // opts: host, token, WebSocket, storage (getItem/setItem, for the schema), + // setTimeout/clearTimeout, opTimeoutMs. + constructor(opts) { + const deps = ProtoClient.deps(); + this.Schema = deps.ProtoSchema; + this.host = opts.host; + this.token = opts.token || ''; + this.WebSocket = opts.WebSocket || (typeof WebSocket !== 'undefined' ? WebSocket : null); + this.storage = opts.storage || null; + this.setTimeout = opts.setTimeout || ((fn, ms) => setTimeout(fn, ms)); + this.clearTimeout = opts.clearTimeout || ((t) => clearTimeout(t)); + this.opTimeoutMs = opts.opTimeoutMs || 10000; + this.schema = this.loadSchema() || new this.Schema(); + this.model = new deps.ProtoModel(this.schema); + this.model.onChange((kind, id, item) => this.emit('change', { kind: kind, id: id, item: item })); + this.listeners = {}; + this.pending = new Map(); + this.nextId = 1; + this.ws = null; + this.ready = false; + this.opened = false; + this.stopped = true; + this.retries = 0; + this.reconnectTimer = null; + this.held = []; + this.fw = null; + } + // In the browser the classes are the ones index.js was joined with; in + // Node they are loaded. + static deps() { + if (typeof require === 'function' && typeof module !== 'undefined' && module.exports) + return { ProtoSchema: require('./schema.js').ProtoSchema, ProtoModel: require('./model.js').ProtoModel }; + return { ProtoSchema: ProtoSchema, ProtoModel: ProtoModel }; + } + + // cb(d) for a device event by its name (shadeState, frame ...), and for + // the client's own: open, hello, synced, change, refused (closed before it + // opened: the token may have been refused), closed, versionMismatch and + // error (a refusal the device couldn't tie to an op). + on(name, cb) { (this.listeners[name] = this.listeners[name] || []).push(cb); } + emit(name, d) { for (const cb of this.listeners[name] || []) cb(d); } + + connect() { + this.stopped = false; + this.open(); + } + close() { + this.stopped = true; + if (this.reconnectTimer) this.clearTimeout(this.reconnectTimer); + this.reconnectTimer = null; + if (this.ws) this.ws.close(); + } + // A new token, from a login: the device checks it as the socket opens. + setToken(token) { + this.token = token || ''; + if (this.ws) this.ws.close(); + else if (!this.stopped) this.open(); + } + url() { + const params = []; + if (this.token) params.push(`apikey=${encodeURIComponent(this.token)}`); + if (this.schema.hash) params.push(`schema=${encodeURIComponent(this.schema.hash)}`); + return `ws://${this.host}/ws${params.length ? '?' + params.join('&') : ''}`; + } + open() { + this.reconnectTimer = null; + this.ready = false; + this.opened = false; + this.held = []; + const ws = new this.WebSocket(this.url()); + this.ws = ws; + ws.onopen = () => { + this.opened = true; + this.retries = 0; + this.emit('open'); + }; + ws.onmessage = (evt) => this.receive(evt.data); + ws.onclose = () => { + if (this.ws !== ws) return; + this.ws = null; + this.ready = false; + for (const [, p] of this.pending) { + this.clearTimeout(p.timer); + p.reject(new ProtoError('closed', 'The connection closed')); + } + this.pending.clear(); + this.emit(this.opened ? 'closed' : 'refused'); + if (this.stopped) return; + // Soon after a close, slower after each failure, at most 10 s. + const ms = Math.min(500 * Math.pow(2, this.retries++), 10000); + this.reconnectTimer = this.setTimeout(() => this.open(), ms); + }; + } + send(type, d, id) { + const frame = { v: ProtoClient.VERSION, t: this.schema.typeNumbers[type] }; + if (typeof frame.t === 'undefined') throw new ProtoError('unknownOp', `The device has no message ${type}`); + if (id) frame.id = id; + if (d) frame.d = d; + this.ws.send(JSON.stringify(frame)); + } + + // Sends an op, or a control message that wants a reply, by its name with + // its fields by name. Resolves with the reply's fields, rejects with a + // ProtoError. + op(name, fields) { + return new Promise((resolve, reject) => { + if (!this.ready || !this.ws) { + reject(new ProtoError('notConnected', 'Not connected to the device')); + return; + } + const id = this.nextId++; + try { + this.send(name, this.schema.pack(fields || {}), id); + } catch (err) { + reject(err instanceof ProtoError ? err : new ProtoError('badRequest', err.message)); + return; + } + const timer = this.setTimeout(() => { + this.pending.delete(id); + reject(new ProtoError('timeout', `No reply to ${name}`)); + }, this.opTimeoutMs); + this.pending.set(id, { resolve: resolve, reject: reject, timer: timer }); + }); + } + subscribe(feed) { return this.op('subscribe', { feed: this.schema.feedNumbers[feed] }); } + unsubscribe(feed) { return this.op('unsubscribe', { feed: this.schema.feedNumbers[feed] }); } + // The shade model and status again, without the schema. + resync() { + this.model.beginSync(); + return this.op('sync'); + } + + receive(text) { + let frame; + try { + frame = JSON.parse(text); + } catch (err) { + return; + } + if (frame.v !== ProtoClient.VERSION) { + this.emit('versionMismatch', { device: frame.v, client: ProtoClient.VERSION }); + return; + } + const d = frame.d || {}; + // hello, the sync and syncEnd have fixed numbers: they come before the + // schema. + switch (frame.t) { + case 1: this.hello(d); return; + case 4: this.syncFrame(d); return; + case 5: this.syncEnd(); return; + default: break; + } + if (!this.ready) this.held.push(frame); + else this.dispatch(frame); + } + hello(d) { + this.fw = d.fw; + // A schema cached for another firmware is no use: the device sends the + // one it has, as it didn't get its hash. + if (this.schema.hash !== d.sh) { + this.schema = new this.Schema(); + this.model.schema = this.schema; + } + this.schema.hash = d.sh; + this.model.beginSync(); + this.emit('hello', d); + } + syncFrame(d) { + this.schema.add(d); + const device = {}; + for (const key of Object.keys(d)) if (!['T', 'F', 'E', 'Q'].includes(key)) device[key] = d[key]; + // The schema's rows come first, so a frame with the device's sections + // has the whole of it. + if (Object.keys(device).length) this.model.applySync(this.schema.expand(device)); + this.ws.send(JSON.stringify({ v: ProtoClient.VERSION, t: 3 })); + } + syncEnd() { + this.model.endSync(); + if (!this.ready) { + this.ready = true; + this.saveSchema(); + const held = this.held; + this.held = []; + for (const frame of held) this.dispatch(frame); + } + this.emit('synced', this.model); + } + dispatch(frame) { + const d = this.schema.expand(frame.d || {}); + if (frame.t === 2) { + const p = typeof frame.id !== 'undefined' ? this.pending.get(frame.id) : null; + if (!p) { + if (!d.ok) this.emit('error', new ProtoError(this.schema.errors[d.code] || 'internal', d.desc, d)); + return; + } + this.pending.delete(frame.id); + this.clearTimeout(p.timer); + if (d.ok) p.resolve(d); + else p.reject(new ProtoError(this.schema.errors[d.code] || 'internal', d.desc, d)); + return; + } + const name = this.schema.types[frame.t]; + if (!name) return; + this.model.applyEvent(name, d); + this.emit(name, d); + } + + loadSchema() { + if (!this.storage) return null; + try { + return this.Schema.load(JSON.parse(this.storage.getItem(ProtoClient.SCHEMA_KEY))); + } catch (err) { + return null; + } + } + saveSchema() { + if (!this.storage) return; + try { + this.storage.setItem(ProtoClient.SCHEMA_KEY, JSON.stringify(this.schema.save())); + } catch (err) { /* Private windows may have no storage: the device sends the schema each time. */ } + } +} +ProtoClient.VERSION = 2; +ProtoClient.SCHEMA_KEY = 'espsomfyProtoSchema'; +if (typeof module !== 'undefined' && module.exports) module.exports = { ProtoClient, ProtoError }; diff --git a/data-src/js/model.js b/data-src/js/model.js new file mode 100644 index 0000000..2ab15e4 --- /dev/null +++ b/data-src/js/model.js @@ -0,0 +1,150 @@ +// The shade model as socket protocol 2 keeps a client up to date with it +// (docs/PROTOCOL.md): rooms, shades, groups and repeaters from the sync, then +// changed by the device's events. Items are inserted or updated by id, so an +// event during the sync does no harm. It takes frames already expanded to +// names (ProtoSchema.expand) and fills in what a frame leaves out at its +// default. The minify step joins this file into index.js; Node loads it on +// its own for the tests (test/js). +class ProtoModel { + constructor(schema) { + this.schema = schema; + this.rooms = new Map(); + this.shades = new Map(); + this.groups = new Map(); + this.repeaters = []; + this.header = {}; + this.status = {}; + this.listeners = []; + this.seen = null; + } + // cb(kind, id, item): kind is room, shade, group, roomRemoved, + // shadeRemoved, groupRemoved, order, repeaters, header or status. + onChange(cb) { this.listeners.push(cb); } + changed(kind, id, item) { for (const cb of this.listeners) cb(kind, id, item); } + + // A sync: the items it doesn't list are gone once it ends. + beginSync() { this.seen = { rooms: new Set(), shades: new Set(), groups: new Set() }; } + applySync(d) { + if (d.header) { + this.header = this.schema.fill(Object.assign({}, d.header), ProtoModel.FIELDS.header); + this.changed('header', 0, this.header); + } + for (const r of d.rooms || []) this.putRoom(r); + for (const s of d.shades || []) this.putShade(s); + for (const g of d.groups || []) this.putGroup(g); + if (d.repeaters) this.putRepeaters(d.repeaters); + if (d.status) { + this.putStatus(d.status, ProtoModel.FIELDS.memory); + this.putStatus(d.status, this.header.connType === 'Ethernet' ? ProtoModel.FIELDS.ethernet : ProtoModel.FIELDS.wifi); + } + } + endSync() { + if (!this.seen) return; + const lists = [['rooms', 'roomRemoved'], ['shades', 'shadeRemoved'], ['groups', 'groupRemoved']]; + for (const [list, removed] of lists) { + for (const id of [...this[list].keys()]) { + if (this.seen[list].has(id)) continue; + this[list].delete(id); + this.changed(removed, id); + } + } + this.seen = null; + } + // An event by its type's name; false for one that isn't about the model. + applyEvent(type, d) { + switch (type) { + case 'room': this.putRoom(d); return true; + case 'shade': this.putShade(d); return true; + case 'group': this.putGroup(d); return true; + case 'shadeState': return this.mergeState(this.shades, 'shade', d.shadeId, d, ProtoModel.FIELDS.shadeState); + case 'groupState': return this.mergeState(this.groups, 'group', d.groupId, d, ProtoModel.FIELDS.groupState); + case 'roomRemoved': return this.remove(this.rooms, type, d.roomId || 0); + case 'shadeRemoved': return this.remove(this.shades, type, d.shadeId || 0); + case 'groupRemoved': return this.remove(this.groups, type, d.groupId || 0); + case 'roomOrder': return this.order(this.rooms, d.ids || []); + case 'shadeOrder': return this.order(this.shades, d.ids || []); + case 'groupOrder': return this.order(this.groups, d.ids || []); + case 'repeaters': this.putRepeaters(d.repeaters || []); return true; + case 'memory': this.putStatus(d, ProtoModel.FIELDS.memory); return true; + case 'wifi': this.putStatus(d, ProtoModel.FIELDS.wifi); return true; + case 'ethernet': this.putStatus(d, ProtoModel.FIELDS.ethernet); return true; + default: return false; + } + } + + // The lists in their sort order. + roomList() { return ProtoModel.sorted(this.rooms); } + shadeList() { return ProtoModel.sorted(this.shades); } + groupList() { return ProtoModel.sorted(this.groups); } + static sorted(map) { return [...map.values()].sort((a, b) => a.sortOrder - b.sortOrder); } + + putRoom(r) { this.put(this.rooms, 'room', r.roomId, r, ProtoModel.FIELDS.room, 'rooms'); } + putShade(s) { + // Its linked remotes leave out their defaults too. + const linked = (s.linkedRemotes || []).map((r) => this.schema.fill(Object.assign({}, r), ProtoModel.FIELDS.linkedRemote)); + this.put(this.shades, 'shade', s.shadeId, Object.assign({}, s, { linkedRemotes: linked }), ProtoModel.FIELDS.shade, 'shades'); + } + putGroup(g) { this.put(this.groups, 'group', g.groupId, g, ProtoModel.FIELDS.group, 'groups'); } + // A whole item: what it had before is replaced. + put(map, kind, id, item, fields, list) { + id = id || 0; + const whole = this.schema.fill(Object.assign({}, item), fields); + map.set(id, whole); + if (this.seen) this.seen[list].add(id); + this.changed(kind, id, whole); + } + // Part of an item: only its state. A state of an item not synced yet + // waits for the item. + mergeState(map, kind, id, d, fields) { + const item = map.get(id || 0); + if (!item) return false; + Object.assign(item, this.schema.fill(Object.assign({}, d), fields)); + this.changed(kind, item[`${kind}Id`], item); + return true; + } + remove(map, kind, id) { + if (!map.delete(id)) return false; + this.changed(kind, id); + return true; + } + order(map, ids) { + ids.forEach((id, i) => { + const item = map.get(id); + if (item) item.sortOrder = i; + }); + this.changed('order', 0, ids); + return true; + } + putRepeaters(list) { + this.repeaters = list.slice(); + this.changed('repeaters', 0, this.repeaters); + } + putStatus(d, fields) { + const part = {}; + for (const name of fields) part[name] = typeof d[name] === 'undefined' ? this.schema.defaultOf(name) : d[name]; + Object.assign(this.status, part); + this.changed('status', 0, this.status); + } +} +// The fields of each kind of item, as the device writes them +// (src/ProtoSync.cpp, src/outputs/ProtoSocket.cpp): what a frame that leaves +// one out means by it. +ProtoModel.FIELDS = { + header: ['maxRooms', 'maxShades', 'maxGroups', 'maxGroupedShades', 'maxLinkedRemotes', 'startingAddress', 'bitLength', + 'proto', 'serverId', 'hostname', 'model', 'chipModel', 'fw', 'connType'], + room: ['roomId', 'name', 'sortOrder'], + shadeState: ['shadeId', 'position', 'target', 'direction', 'myPos', 'tiltPosition', 'tiltTarget', 'tiltDirection', + 'myTiltPos', 'flags', 'lastRollingCode'], + groupState: ['groupId', 'flags', 'direction'], + linkedRemote: ['remoteAddress', 'lastRollingCode'], + group: ['groupId', 'roomId', 'name', 'remoteAddress', 'lastRollingCode', 'bitLength', 'proto', 'sunSensor', + 'flipCommands', 'flags', 'repeats', 'sortOrder', 'linkedShades'], + memory: ['freeHeap', 'maxHeap', 'minHeap', 'totalHeap', 'uptime', 'loopRate', 'loopMaxUs'], + wifi: ['ssid', 'strength', 'channel'], + ethernet: ['connected', 'speed', 'fullDuplex'] +}; +ProtoModel.FIELDS.shade = ProtoModel.FIELDS.shadeState.concat(['roomId', 'name', 'remoteAddress', 'upTime', 'downTime', + 'tiltTime', 'stepSize', 'paired', 'tiltType', 'shadeType', 'bitLength', 'proto', 'flipCommands', 'flipPosition', + 'inGroup', 'sunSensor', 'light', 'simMy', 'repeats', 'sortOrder', 'gpioUp', 'gpioDown', 'gpioMy', 'gpioLLTrigger', + 'linkedRemotes']); +if (typeof module !== 'undefined' && module.exports) module.exports = { ProtoModel }; diff --git a/data-src/js/schema.js b/data-src/js/schema.js new file mode 100644 index 0000000..50e34ea --- /dev/null +++ b/data-src/js/schema.js @@ -0,0 +1,96 @@ +// Socket protocol 2's schema (docs/PROTOCOL.md): the device's table of message +// types, fields, error codes and feeds, as the sync sends it. It turns the +// short keys of a frame into names and back. The minify step joins this file +// into index.js; Node loads it on its own for the tests (test/js). +class ProtoSchema { + constructor() { + this.hash = null; + this.types = {}; // number -> name + this.typeNumbers = {}; // name -> number + this.fields = {}; // key -> { name, kind, def } + this.keys = {}; // name -> key + this.errors = {}; + this.errorNumbers = {}; + this.feeds = {}; + this.feedNumbers = {}; + this.rows = { T: [], F: [], E: [], Q: [] }; + } + // The schema's rows in a sync frame's payload: T types, F fields, E error + // codes, Q feeds. They may span several frames. + add(d) { + for (const [number, name] of d.T || []) { + this.types[number] = name; + this.typeNumbers[name] = number; + } + for (const [key, name, kind, def] of d.F || []) { + this.fields[key] = { name: name, kind: kind, def: def }; + this.keys[name] = key; + } + for (const [number, name] of d.E || []) { + this.errors[number] = name; + this.errorNumbers[name] = number; + } + for (const [number, name] of d.Q || []) { + this.feeds[number] = name; + this.feedNumbers[name] = number; + } + for (const k of ['T', 'F', 'E', 'Q']) if (d[k]) this.rows[k].push(...d[k]); + } + get empty() { return Object.keys(this.fields).length === 0; } + // What a field left out of a frame is: its default. + defaultOf(name) { + const f = this.fields[this.keys[name]]; + if (!f) return undefined; + switch (f.kind) { + case 'b': return f.def !== 0; + case 's': return ''; + case 'l': return []; + case 'o': return {}; + default: return f.def; + } + } + // Short keys to names, and 1/0 to true/false, all the way down. A field + // left out stays out; fill() puts the defaults in. + expand(value) { + if (Array.isArray(value)) return value.map((v) => this.expand(v)); + if (value === null || typeof value !== 'object') return value; + const out = {}; + for (const key of Object.keys(value)) { + const f = this.fields[key]; + let v = value[key]; + if (f && f.kind === 'b' && typeof v === 'number') v = v !== 0; + out[f ? f.name : key] = this.expand(v); + } + return out; + } + // Names to short keys, and booleans to 1/0, for an op's payload. A name + // the schema doesn't have is an error: the device would refuse it anyway. + pack(value) { + if (Array.isArray(value)) return value.map((v) => this.pack(v)); + if (value === null || typeof value !== 'object') return typeof value === 'boolean' ? (value ? 1 : 0) : value; + const out = {}; + for (const name of Object.keys(value)) { + if (typeof value[name] === 'undefined') continue; + const key = this.keys[name]; + if (typeof key === 'undefined') throw new Error(`The device has no field ${name}`); + out[key] = this.pack(value[name]); + } + return out; + } + // Sets each of names that obj lacks to its default, as a frame leaves out + // a field at its default. Returns obj. + fill(obj, names) { + for (const name of names) if (typeof obj[name] === 'undefined') obj[name] = this.defaultOf(name); + return obj; + } + // What a cache keeps, and the schema back from it. + save() { return { hash: this.hash, rows: this.rows }; } + static load(saved) { + if (!saved || typeof saved.hash !== 'string' || !saved.rows) return null; + const s = new ProtoSchema(); + s.add(saved.rows); + s.hash = saved.hash; + return s.empty ? null : s; + } +} +if (typeof module !== 'undefined' && module.exports) module.exports = { ProtoSchema }; diff --git a/data-src/login.html b/data-src/login.html index 32623f1..c789f6c 100644 --- a/data-src/login.html +++ b/data-src/login.html @@ -3,9 +3,7 @@ - - - + diff --git a/minify.py b/minify.py index a5c581c..f7b48d6 100644 --- a/minify.py +++ b/minify.py @@ -52,6 +52,16 @@ MINIFY_AND_GZIP = {".html", ".htm", ".css", ".js", ".json", ".svg", ".xml"} # Extensions that optipng can optimise PNG_EXTENSIONS = {".png"} +# Files joined into one, in this order, before they are minified: a page load +# then opens fewer connections, each of which costs the device heap +# (docs/adr/0005-one-websocket-protocol.md). The protocol 2 client in js/ +# comes before index.js, which uses it. A source listed here isn't copied on +# its own. +BUNDLES = { + "index.js": ["js/schema.js", "js/model.js", "js/client.js", "index.js"], + "app.css": ["main.css", "widgets.css", "icons.css"], +} + # Everything else is copied verbatim # ────────────────────────────────────────────── @@ -252,6 +262,33 @@ def minify_all(): total_final = 0 file_count = 0 + def report(rel_path, stats): + nonlocal total_original, total_final, file_count + pct = (1 - stats["final"] / stats["original"]) * 100 if stats["original"] > 0 else 0 + print( + f" {rel_path:<40s} " + f"{stats['original']:>8,d} → {stats['final']:>8,d} B " + f"({pct:+.0f}%) [{stats['action']}]" + ) + total_original += stats["original"] + total_final += stats["final"] + file_count += 1 + + bundled = set() + work_dir = os.path.join(dst_dir, ".bundles") + os.makedirs(work_dir, exist_ok=True) + for name, parts in BUNDLES.items(): + joined = os.path.join(work_dir, name) + with open(joined, "w", encoding="utf-8") as out: + for part in parts: + with open(os.path.join(src_dir, part), "r", encoding="utf-8") as f: + out.write(f.read()) + # A file without a final newline would run into the next. + out.write("\n") + bundled.add(os.path.normpath(part)) + report(f"{name} ({len(parts)} files)", process_file(joined, os.path.join(dst_dir, name))) + shutil.rmtree(work_dir) + for root, dirs, files in os.walk(src_dir): for fname in sorted(files): # Skip hidden files and editor temp files @@ -260,20 +297,11 @@ def minify_all(): src_path = os.path.join(root, fname) rel_path = os.path.relpath(src_path, src_dir) + if os.path.normpath(rel_path) in bundled: + continue dst_path = os.path.join(dst_dir, rel_path) - stats = process_file(src_path, dst_path) - - pct = (1 - stats["final"] / stats["original"]) * 100 if stats["original"] > 0 else 0 - print( - f" {rel_path:<40s} " - f"{stats['original']:>8,d} → {stats['final']:>8,d} B " - f"({pct:+.0f}%) [{stats['action']}]" - ) - - total_original += stats["original"] - total_final += stats["final"] - file_count += 1 + report(rel_path, process_file(src_path, dst_path)) print(f"[minify] {file_count} files processed") print( diff --git a/test/e2e/test_survival.py b/test/e2e/test_survival.py index 457e40e..a6dcae9 100644 --- a/test/e2e/test_survival.py +++ b/test/e2e/test_survival.py @@ -54,7 +54,7 @@ def test_one_client_too_many_is_closed(connect, schema): def test_page_load(host, token, connect, watch_heap): """The static files in parallel, as a browser fetches them, then the socket.""" - files = ["/", "/main.css", "/widgets.css", "/icons.css", "/index.js", "/icon.png"] + files = ["/", "/app.css", "/index.js", "/icon.png"] errors = [] def get(path): diff --git a/test/js/client.test.js b/test/js/client.test.js new file mode 100644 index 0000000..17d5c39 --- /dev/null +++ b/test/js/client.test.js @@ -0,0 +1,200 @@ +// The web UI's ProtoClient (data-src/js/client.js): the recorded hello and +// sync replayed through a stand-in socket. +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { ProtoClient, ProtoError } = require('../../data-src/js/client.js'); +const { hello, sync, recordedItems, recordedSchema, FakeSocket, FakeTimers, FakeStorage } = require('./helpers.js'); + +function client(opts) { + FakeSocket.reset(); + const timers = new FakeTimers(); + const c = new ProtoClient(Object.assign({ host: 'device', WebSocket: FakeSocket, setTimeout: timers.setTimeout, + clearTimeout: timers.clearTimeout }, opts || {})); + c.timers = timers; + return c; +} +// Connects and plays the recorded hello and sync. +function connected(opts) { + const c = client(opts); + c.connect(); + const ws = FakeSocket.last; + ws.open(); + ws.deliver(hello); + for (const f of sync) ws.deliver(f); + return { c, ws }; +} +const schema = recordedSchema(); +const T = schema.typeNumbers; +function reply(id, d) { return { v: 2, t: T.reply, id: id, d: d }; } + +test('the client connects with its token and syncs, sending next after each sync frame', () => { + const c = client({ token: 'abc 123' }); + c.connect(); + const ws = FakeSocket.last; + assert.equal(ws.url, 'ws://device/ws?apikey=abc%20123'); + ws.open(); + ws.deliver(hello); + for (const f of sync) ws.deliver(f); + const syncFrames = sync.filter((f) => f.t === T.sync).length; + assert.equal(ws.sent.filter((f) => f.t === T.next).length, syncFrames); + assert.ok(c.ready); + assert.equal(c.fw, hello.d.fw); + assert.equal(c.model.shades.size, recordedItems('S').length); +}); + +test('synced comes once the sync has ended', () => { + const c = client(); + let synced = 0; + c.on('synced', () => synced++); + c.connect(); + const ws = FakeSocket.last; + ws.open(); + ws.deliver(hello); + for (const f of sync.filter((f) => f.t === T.sync)) ws.deliver(f); + assert.equal(synced, 0); + ws.deliver(sync.find((f) => f.t === T.syncEnd)); + assert.equal(synced, 1); +}); + +test('an op resolves with its reply', async () => { + const { c, ws } = connected(); + const p = c.op('getSettings', { section: 'radio' }); + const sent = ws.sent[ws.sent.length - 1]; + assert.deepEqual(sent, { v: 2, t: T.getSettings, id: sent.id, d: { se: 'radio' } }); + ws.deliver(reply(sent.id, { k: 1, bl: 80, en: 1 })); + assert.deepEqual(await p, { ok: true, bitLength: 80, enabled: true }); +}); + +test('a refused op rejects with the error by its name', async () => { + const { c, ws } = connected(); + const p = c.op('saveSettings', { section: 'mqtt' }); + const id = ws.sent[ws.sent.length - 1].id; + ws.deliver(reply(id, { e: schema.errorNumbers.badRequest, x: 'Nothing to save' })); + await assert.rejects(p, (err) => err instanceof ProtoError && err.code === 'badRequest' && err.desc === 'Nothing to save'); +}); + +test('an op is refused before the sync, and when its reply never comes', async () => { + const c = client(); + await assert.rejects(c.op('reboot'), (err) => err.code === 'notConnected'); + const { c: c2 } = connected(); + const p = c2.op('reboot'); + c2.timers.runAll(); + await assert.rejects(p, (err) => err.code === 'timeout'); +}); + +test('an op of a name or field the device lacks is refused here', async () => { + const { c } = connected(); + await assert.rejects(c.op('noSuchOp'), (err) => err.code === 'unknownOp'); + await assert.rejects(c.op('reboot', { noSuchField: 1 }), (err) => err.code === 'badRequest'); +}); + +test('ops waiting when the socket closes are refused, and the client connects again', async () => { + const { c, ws } = connected(); + const p = c.op('scanWifi'); + let closed = 0; + c.on('closed', () => closed++); + ws.drop(); + await assert.rejects(p, (err) => err.code === 'closed'); + assert.equal(closed, 1); + assert.equal(c.ready, false); + assert.deepEqual(c.timers.runAll(), [500]); + assert.notEqual(FakeSocket.last, ws); +}); + +test('a socket refused before it opened says so, and the retries slow down', () => { + const c = client(); + let refused = 0; + c.on('refused', () => refused++); + c.connect(); + const delays = []; + for (let i = 0; i < 6; i++) { + FakeSocket.last.drop(); + delays.push(...c.timers.runAll()); + } + assert.equal(refused, 6); + assert.deepEqual(delays, [500, 1000, 2000, 4000, 8000, 10000]); +}); + +test('close stops the reconnecting', () => { + const { c } = connected(); + c.close(); + assert.deepEqual(c.timers.runAll(), []); +}); + +test('events update the model, and are told by name', () => { + const { c, ws } = connected(); + const shade = c.model.shadeList()[0]; + const seen = []; + c.on('shadeState', (d) => seen.push(d)); + ws.deliver({ v: 2, t: T.shadeState, d: { s: shade.shadeId, t: 30, d: 1 } }); + assert.equal(c.model.shades.get(shade.shadeId).target, 30); + assert.deepEqual(seen, [{ shadeId: shade.shadeId, target: 30, direction: 1 }]); +}); + +test('an event before the schema is kept until the sync ends', () => { + const c = client(); + c.connect(); + const ws = FakeSocket.last; + ws.open(); + ws.deliver(hello); + // Before the schema: the client can't read it yet. + const shadeId = recordedItems('S')[0].s; + ws.deliver({ v: 2, t: T.shadeState, d: { s: shadeId, t: 55 } }); + for (const f of sync) ws.deliver(f); + assert.equal(c.model.shades.get(shadeId).target, 55); +}); + +test('the schema is kept, and the device is told it has it', () => { + const storage = new FakeStorage(); + connected({ storage: storage }); + const c = client({ storage: storage }); + c.connect(); + const ws = FakeSocket.last; + assert.equal(ws.url, `ws://device/ws?schema=${hello.d.sh}`); + ws.open(); + ws.deliver(hello); + // Without the schema's rows, as the device sends a client that has it. + for (const f of sync) { + const d = Object.assign({}, f.d); + for (const k of ['T', 'F', 'E', 'Q']) delete d[k]; + ws.deliver(Object.assign({}, f, { d: d })); + } + assert.ok(c.ready); + assert.equal(c.model.shades.size, recordedItems('S').length); +}); + +test('a kept schema of another firmware is dropped', () => { + const storage = new FakeStorage(); + connected({ storage: storage }); + const c = client({ storage: storage }); + c.connect(); + const ws = FakeSocket.last; + ws.open(); + ws.deliver({ v: 2, t: 1, d: { fw: 'v4.0.0', sh: 'ffffffff' } }); + assert.equal(c.schema.hash, 'ffffffff'); + assert.ok(c.schema.empty); +}); + +test('a frame of another protocol version is reported', () => { + const { c, ws } = connected(); + let mismatch = null; + c.on('versionMismatch', (d) => { mismatch = d; }); + ws.deliver({ v: 3, t: 1, d: {} }); + assert.deepEqual(mismatch, { device: 3, client: 2 }); +}); + +test('a new token connects again with it', () => { + const { c, ws } = connected(); + c.setToken('new'); + assert.ok(ws.closed); + c.timers.runAll(); + assert.match(FakeSocket.last.url, /apikey=new/); +}); + +test('subscribe sends the feed number', () => { + const { c, ws } = connected(); + c.subscribe('frames'); + const sent = ws.sent[ws.sent.length - 1]; + assert.equal(sent.t, T.subscribe); + assert.deepEqual(sent.d, { fe: schema.feedNumbers.frames }); +}); diff --git a/test/js/helpers.js b/test/js/helpers.js new file mode 100644 index 0000000..fd73718 --- /dev/null +++ b/test/js/helpers.js @@ -0,0 +1,82 @@ +// What the web UI client tests share: the frames the e2e suite recorded from a +// device (test/fixtures/proto2), and stand-ins for the socket and the timers. +const path = require('node:path'); + +const FIXTURES = path.join(__dirname, '..', 'fixtures', 'proto2'); +const hello = require(path.join(FIXTURES, 'hello.json')); +const sync = require(path.join(FIXTURES, 'sync.json')); +const schemaJson = require(path.join(FIXTURES, 'schema.json')); +const { ProtoSchema } = require('../../data-src/js/schema.js'); + +// A schema filled from the recorded sync, as a client builds it. +function recordedSchema() { + const s = new ProtoSchema(); + for (const f of sync) if (f.t === 4) s.add(f.d); + s.hash = hello.d.sh; + return s; +} +// The recorded sync's device sections, expanded, frame by frame. +function deviceSections(schema) { + return sync.filter((f) => f.t === 4).map((f) => { + const d = {}; + for (const k of Object.keys(f.d)) if (!['T', 'F', 'E', 'Q'].includes(k)) d[k] = f.d[k]; + return schema.expand(d); + }); +} +// The recorded sync's items under their short keys. +function recordedItems(key) { + return sync.filter((f) => f.t === 4 && f.d[key]).flatMap((f) => f.d[key]); +} + +// A WebSocket the test drives: it records what the client sends, and the test +// opens, delivers and closes. +class FakeSocket { + constructor(url) { + this.url = url; + this.sent = []; + this.closed = false; + FakeSocket.last = this; + FakeSocket.all.push(this); + } + send(text) { this.sent.push(JSON.parse(text)); } + close() { + if (this.closed) return; + this.closed = true; + if (this.onclose) this.onclose({}); + } + open() { if (this.onopen) this.onopen({}); } + deliver(frame) { this.onmessage({ data: typeof frame === 'string' ? frame : JSON.stringify(frame) }); } + // The device closing it, or refusing it before it opened. + drop() { this.close(); } +} +FakeSocket.all = []; +FakeSocket.reset = () => { FakeSocket.all = []; FakeSocket.last = null; }; + +// Timers the test runs by hand. +class FakeTimers { + constructor() { + this.timers = new Map(); + this.next = 1; + this.setTimeout = (fn, ms) => { + const id = this.next++; + this.timers.set(id, { fn: fn, ms: ms }); + return id; + }; + this.clearTimeout = (id) => { this.timers.delete(id); }; + } + // Runs every timer waiting, and returns their delays. + runAll() { + const due = [...this.timers.entries()]; + this.timers.clear(); + for (const [, t] of due) t.fn(); + return due.map(([, t]) => t.ms); + } +} +// localStorage as far as the client uses it. +class FakeStorage { + constructor() { this.items = {}; } + getItem(key) { return Object.prototype.hasOwnProperty.call(this.items, key) ? this.items[key] : null; } + setItem(key, value) { this.items[key] = String(value); } +} + +module.exports = { hello, sync, schemaJson, recordedSchema, deviceSections, recordedItems, FakeSocket, FakeTimers, FakeStorage }; diff --git a/test/js/model.test.js b/test/js/model.test.js new file mode 100644 index 0000000..a7dc239 --- /dev/null +++ b/test/js/model.test.js @@ -0,0 +1,106 @@ +// The web UI's ProtoModel (data-src/js/model.js): the recorded sync, then +// events. +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { ProtoModel } = require('../../data-src/js/model.js'); +const { recordedSchema, deviceSections, recordedItems } = require('./helpers.js'); + +function synced() { + const schema = recordedSchema(); + const model = new ProtoModel(schema); + model.beginSync(); + for (const d of deviceSections(schema)) model.applySync(d); + model.endSync(); + return model; +} + +test('the sync fills every room, shade and group', () => { + const m = synced(); + assert.equal(m.rooms.size, recordedItems('R').length); + assert.equal(m.shades.size, recordedItems('S').length); + assert.equal(m.groups.size, recordedItems('G').length); + assert.equal(m.header.model, 'ESPSomfyRTS'); + assert.equal(m.header.maxShades, 32); + assert.ok(m.status.freeHeap > 0); +}); + +test('a field the sync left out is its default', () => { + const m = synced(); + const raw = recordedItems('S').find((s) => typeof s.m === 'undefined'); + const shade = m.shades.get(raw.s); + // Left out because they were at their default. + assert.equal(shade.myPos, -1); + assert.equal(shade.stepSize, raw.z === undefined ? 100 : raw.z); + assert.equal(typeof shade.paired, 'boolean'); + assert.ok(Array.isArray(shade.linkedRemotes)); + for (const r of shade.linkedRemotes) assert.equal(typeof r.lastRollingCode, 'number'); + for (const g of m.groupList()) assert.ok(Array.isArray(g.linkedShades)); +}); + +test('the lists come in their sort order', () => { + const m = synced(); + const orders = m.shadeList().map((s) => s.sortOrder); + assert.deepEqual(orders, [...orders].sort((a, b) => a - b)); +}); + +test('a state event changes only the state, and a left-out field is its default', () => { + const m = synced(); + const shade = m.shadeList()[0]; + const name = shade.name; + // A shade at rest at the top: position 0 is left out. + assert.ok(m.applyEvent('shadeState', { shadeId: shade.shadeId, target: 40, direction: 1 })); + assert.equal(m.shades.get(shade.shadeId).position, 0); + assert.equal(m.shades.get(shade.shadeId).target, 40); + assert.equal(m.shades.get(shade.shadeId).name, name); + // Of a shade the client doesn't have yet: nothing to change. + assert.equal(m.applyEvent('shadeState', { shadeId: 250, position: 5 }), false); +}); + +test('a whole item replaces what was there', () => { + const m = synced(); + const shade = m.shadeList()[0]; + m.applyEvent('shade', { shadeId: shade.shadeId, name: 'Renamed' }); + assert.equal(m.shades.get(shade.shadeId).name, 'Renamed'); + assert.equal(m.shades.get(shade.shadeId).upTime, 0); + m.applyEvent('room', { roomId: 9, name: 'New' }); + assert.equal(m.rooms.get(9).sortOrder, 0); +}); + +test('removals, orders and repeaters', () => { + const m = synced(); + const changes = []; + m.onChange((kind, id) => changes.push([kind, id])); + const [first, second] = m.shadeList(); + assert.ok(m.applyEvent('shadeRemoved', { shadeId: first.shadeId })); + assert.equal(m.shades.has(first.shadeId), false); + // An id it doesn't have is ignored. + assert.equal(m.applyEvent('roomRemoved', { roomId: 250 }), false); + m.applyEvent('shadeOrder', { ids: [second.shadeId] }); + assert.equal(m.shades.get(second.shadeId).sortOrder, 0); + m.applyEvent('repeaters', { repeaters: [7, 8] }); + assert.deepEqual(m.repeaters, [7, 8]); + assert.deepEqual(changes.map((c) => c[0]), ['shadeRemoved', 'order', 'repeaters']); +}); + +test('a sync drops what it no longer lists, and keeps what an event added during it', () => { + const m = synced(); + const schema = m.schema; + m.applyEvent('room', { roomId: 99, name: 'Gone' }); + m.beginSync(); + m.applyEvent('room', { roomId: 98, name: 'Added during the sync' }); + for (const d of deviceSections(schema)) m.applySync(d); + m.endSync(); + assert.equal(m.rooms.has(99), false); + assert.equal(m.rooms.has(98), true); +}); + +test('status events', () => { + const m = synced(); + m.applyEvent('memory', { freeHeap: 1000, uptime: 5 }); + assert.equal(m.status.freeHeap, 1000); + assert.equal(m.status.loopRate, 0); + m.applyEvent('wifi', { strength: -60, channel: 6 }); + assert.equal(m.status.ssid, ''); + assert.equal(m.status.strength, -60); + assert.equal(m.applyEvent('frame', { remoteAddress: 1 }), false); +}); diff --git a/test/js/schema.test.js b/test/js/schema.test.js new file mode 100644 index 0000000..0ec1136 --- /dev/null +++ b/test/js/schema.test.js @@ -0,0 +1,56 @@ +// The web UI's ProtoSchema (data-src/js/schema.js) against the schema a +// device sent (test/fixtures/proto2). +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { ProtoSchema } = require('../../data-src/js/schema.js'); +const { recordedSchema, schemaJson } = require('./helpers.js'); + +test('the schema is collected from every sync frame it spans', () => { + const s = recordedSchema(); + assert.equal(Object.keys(s.types).length, Object.keys(schemaJson.types).length); + assert.equal(Object.keys(s.fields).length, Object.keys(schemaJson.fields).length); + assert.equal(s.typeNumbers.shadeCommand, 16); + assert.equal(s.typeNumbers.saveSettings, 50); + assert.equal(s.keys.shadeId, 's'); + assert.equal(s.errorNumbers.busy, 5); + assert.equal(s.feedNumbers.frequencyScan, 2); +}); + +test('short keys become names, and 1/0 booleans, all the way down', () => { + const s = recordedSchema(); + const shade = s.expand({ s: 3, n: 'Office', i: 1, fp: 0, lk: [{ a: 5, c: 9 }] }); + assert.deepEqual(shade, { shadeId: 3, name: 'Office', paired: true, flipPosition: false, + linkedRemotes: [{ remoteAddress: 5, lastRollingCode: 9 }] }); + // A whole number that happens to be 1 stays a number. + assert.deepEqual(s.expand({ p: 1 }), { position: 1 }); +}); + +test('names become short keys for an op', () => { + const s = recordedSchema(); + assert.deepEqual(s.pack({ shadeId: 3, command: 'Up', paired: true, sunny: false }), { s: 3, cm: 'Up', i: 1, sn: 0 }); + assert.deepEqual(s.pack({ ids: [3, 1, 2] }), { is: [3, 1, 2] }); + // Left out, not sent as undefined. + assert.deepEqual(s.pack({ shadeId: 3, target: undefined }), { s: 3 }); + assert.throws(() => s.pack({ nonsense: 1 }), /no field nonsense/); +}); + +test('a field left out is its default', () => { + const s = recordedSchema(); + assert.equal(s.defaultOf('myPos'), -1); + assert.equal(s.defaultOf('stepSize'), 100); + assert.equal(s.defaultOf('bitLength'), 56); + assert.equal(s.defaultOf('paired'), false); + assert.equal(s.defaultOf('name'), ''); + assert.deepEqual(s.defaultOf('linkedShades'), []); + assert.deepEqual(s.fill({ shadeId: 2 }, ['shadeId', 'position', 'myPos']), { shadeId: 2, position: 0, myPos: -1 }); +}); + +test('a cached schema comes back whole, with its hash', () => { + const s = recordedSchema(); + const back = ProtoSchema.load(JSON.parse(JSON.stringify(s.save()))); + assert.equal(back.hash, s.hash); + assert.deepEqual(back.fields, s.fields); + assert.deepEqual(back.types, s.types); + assert.equal(ProtoSchema.load(null), null); + assert.equal(ProtoSchema.load({ hash: 'x', rows: {} }), null); +});