perf: throttle MQTT DB writes with 2s batched flush

Collect payloads in memory Map, flush to SQLite every 2 seconds
instead of on every message. Prevents DB lock and slowness with
high-volume brokers like Venus OS.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

display/middleware/src/modules/mqtt/broker-manager.ts

@@ -162,33 +162,40 @@ export function connectBroker(id: number): Promise<mqtt.MqttClient> {
       resolve(client);
     });
 
-    client.on('message', (topic: string, payloadBuf: Buffer) => {
+    // Throttle DB writes: cache payloads in memory, flush periodically
+    const payloadCache = new Map<string, string>();
+    const flushPayloads = (): void => {
       const db = getDb();
-      const payload = payloadBuf.toString();
+      for (const [t, p] of payloadCache.entries()) {
-
-      // Find the topic record
         const topicRecord = db
-        .prepare(
+          .prepare('SELECT * FROM mqtt_topics WHERE broker_id = ? AND topic = ?')
-          'SELECT * FROM mqtt_topics WHERE broker_id = ? AND topic = ?'
+          .get(id, t) as MqttTopic | undefined;
-        )
+        if (!topicRecord) continue;
-        .get(id, topic) as MqttTopic | undefined;
 
-      if (!topicRecord) return;
+        db.run('UPDATE mqtt_topics SET last_payload = ? WHERE id = ?', [p, topicRecord.id]);
 
-      // Update last_payload
+        const extracted = extractValue(p, topicRecord.json_path);
-      db.run(
+        if (extracted === null) continue;
-        'UPDATE mqtt_topics SET last_payload = ? WHERE id = ?',
-        [payload, topicRecord.id]
-      );
-
-      // Extract value and update datapoint
-      const extracted = extractValue(payload, topicRecord.json_path);
-      if (extracted === null) return;
 
         const dp = getDatapointByTopicId(topicRecord.id);
         if (dp) {
           updateDatapointValue(dp.id, extracted);
         }
+      }
+      payloadCache.clear();
+    };
+
+    // Flush every 2 seconds
+    const flushTimer = setInterval(flushPayloads, 2000);
+
+    client.on('message', (topic: string, payloadBuf: Buffer) => {
+      payloadCache.set(topic, payloadBuf.toString());
+    });
+
+    // Clean up flush timer on disconnect
+    client.on('close', () => {
+      clearInterval(flushTimer);
+      flushPayloads();
     });
 
     client.on('error', (err: Error) => {

display/middleware/src/modules/mqtt/topic-discovery.ts

@@ -87,24 +87,37 @@ export function startDiscovery(brokerId: number, prefix?: string): Promise<MqttT
     const discoveredTopics: Set<string> = new Set();
 
     const finalize = (): void => {
+      clearInterval(flushInterval);
+      flushToDb(); // Final flush
       if (tempClient) {
         tempClient.end(true);
       }
       resolve(getTopicsForBroker(brokerId));
     };
 
-    const onMessage = (topic: string, payload: Buffer): void => {
+    // Collect payloads in memory, batch-write to DB periodically
-      const payloadStr = payload.toString();
+    const payloadCache: Map<string, string> = new Map();
-      if (!discoveredTopics.has(topic)) {
+
-        discoveredTopics.add(topic);
+    const flushToDb = (): void => {
-        ensureTopicExists(brokerId, topic);
-      }
-      // Always update last_payload so the user sees the latest value
       const db = getDb();
+      for (const [t, p] of payloadCache.entries()) {
+        if (!discoveredTopics.has(t)) {
+          discoveredTopics.add(t);
+          ensureTopicExists(brokerId, t);
+        }
         db.run(
           'UPDATE mqtt_topics SET last_payload = ? WHERE broker_id = ? AND topic = ?',
-        [payloadStr, brokerId, topic]
+          [p, brokerId, t]
         );
+      }
+      payloadCache.clear();
+    };
+
+    // Flush every 2 seconds instead of on every message
+    const flushInterval = setInterval(flushToDb, 2000);
+
+    const onMessage = (topic: string, payload: Buffer): void => {
+      payloadCache.set(topic, payload.toString());
     };
 
     if (existingClient && existingClient.connected) {