From e6b366862cfe3e9bd76c90238ecbcc59937561ab Mon Sep 17 00:00:00 2001 From: Christian Mueller Date: Tue, 7 Apr 2026 04:19:54 +0200 Subject: [PATCH] perf: server-side search + pagination for topics (30 per page) Loads only 30 topics at a time with server-side search instead of fetching all 6000+ topics. Debounced search input, pagination, selected-only filter. Much faster for large Victron brokers. Co-Authored-By: Claude Opus 4.6 (1M context) --- display/frontend/src/pages/MqttTopics.tsx | 148 ++++++++++-------- .../middleware/src/api/routes/mqtt-topics.ts | 8 +- .../src/modules/mqtt/topic-discovery.ts | 29 +++- 3 files changed, 117 insertions(+), 68 deletions(-) diff --git a/display/frontend/src/pages/MqttTopics.tsx b/display/frontend/src/pages/MqttTopics.tsx index 9efe764..bc504f7 100644 --- a/display/frontend/src/pages/MqttTopics.tsx +++ b/display/frontend/src/pages/MqttTopics.tsx @@ -1,4 +1,4 @@ -import { useEffect, useState } from 'react'; +import { useEffect, useState, useCallback } from 'react'; import { useParams } from 'react-router-dom'; import { api } from '../api/client'; @@ -13,37 +13,57 @@ interface Topic { discovered_at: string; } +interface TopicsResponse { + topics: Topic[]; + total: number; +} + +const PAGE_SIZE = 30; + export default function MqttTopics() { const { brokerId } = useParams<{ brokerId: string }>(); - const [topics, setTopics] = useState([]); - const [filter, setFilter] = useState(''); + const [data, setData] = useState({ topics: [], total: 0 }); + const [search, setSearch] = useState(''); + const [searchInput, setSearchInput] = useState(''); + const [showSelected, setShowSelected] = useState(false); + const [page, setPage] = useState(0); const [prefix, setPrefix] = useState(''); const [error, setError] = useState(''); const [discoveryMsg, setDiscoveryMsg] = useState(''); const [editPaths, setEditPaths] = useState>({}); - async function load() { + const load = useCallback(async () => { try { - const data = await api.get(`/api/mqtt/brokers/${brokerId}/topics`); - setTopics(data); + const params = new URLSearchParams(); + if (search) params.set('search', search); + if (showSelected) params.set('selected', '1'); + params.set('limit', String(PAGE_SIZE)); + params.set('offset', String(page * PAGE_SIZE)); + const result = await api.get(`/api/mqtt/brokers/${brokerId}/topics?${params}`); + setData(result); const paths: Record = {}; - data.forEach((t) => { paths[t.id] = t.json_path ?? ''; }); - setEditPaths(paths); + result.topics.forEach((t) => { paths[t.id] = t.json_path ?? ''; }); + setEditPaths((prev) => ({ ...prev, ...paths })); } catch (e) { setError((e as Error).message); } - } + }, [brokerId, search, showSelected, page]); - useEffect(() => { load(); }, [brokerId]); + useEffect(() => { load(); }, [load]); + + // Debounced search + useEffect(() => { + const timer = setTimeout(() => { setSearch(searchInput); setPage(0); }, 400); + return () => clearTimeout(timer); + }, [searchInput]); async function handleDiscover() { setDiscoveryMsg(''); try { await api.post(`/api/mqtt/brokers/${brokerId}/topics/discover`, { prefix: prefix || null }); - setDiscoveryMsg('Discovery laeuft 30 Sekunden... Liste wird automatisch aktualisiert.'); - // Refresh every 3 seconds during the 30s discovery window - const interval = setInterval(() => load(), 3000); - setTimeout(() => { clearInterval(interval); setDiscoveryMsg('Discovery abgeschlossen.'); load(); }, 32000); + setDiscoveryMsg('Discovery laeuft 30s...'); + const interval = setInterval(() => load(), 5000); + setTimeout(() => { clearInterval(interval); setDiscoveryMsg(''); load(); }, 32000); } catch (e) { setError((e as Error).message); } @@ -58,9 +78,9 @@ export default function MqttTopics() { } } - async function handleIgnore(id: number, ignored: number) { + async function handleIgnore(id: number) { try { - await api.put(`/api/mqtt/brokers/${brokerId}/topics/${id}`, { ignored: ignored ? 0 : 1 }); + await api.put(`/api/mqtt/brokers/${brokerId}/topics/${id}`, { ignored: 1 }); load(); } catch (e) { setError((e as Error).message); @@ -75,88 +95,92 @@ export default function MqttTopics() { } } - const filtered = topics.filter((t) => - !t.ignored && t.topic.toLowerCase().includes(filter.toLowerCase()) - ); + const totalPages = Math.ceil(data.total / PAGE_SIZE); return (

MQTT Topics

-

Broker ID: {brokerId}

- {error &&
{error}
} +

Broker #{brokerId} — {data.total} Topics gefunden

+ {error &&
{error}
} {/* Controls */} -
+
setFilter(e.target.value)} + placeholder="Topics suchen (z.B. grid, battery, pv)..." + className="border border-gray-300 rounded px-3 py-1.5 text-sm w-72" + value={searchInput} + onChange={(e) => setSearchInput(e.target.value)} /> + +
setPrefix(e.target.value)} /> -
- {discoveryMsg &&

{discoveryMsg}

} + {discoveryMsg &&

{discoveryMsg}

} + {/* Topic list */}
- - - - - + + + + + - {filtered.length === 0 && ( - + {data.topics.length === 0 && ( + )} - {filtered.map((t) => ( + {data.topics.map((t) => ( - - - - + + - ))}
AktivTopicLetzter WertJSON-PfadAktionAktivTopicLetzter WertJSON-PfadAktion
Keine Topics gefunden
+ {search ? 'Keine Topics fuer diese Suche' : 'Keine Topics gefunden. Starte Discovery.'} +
- + handleSelect(t.id, e.target.checked)} - className="w-4 h-4 accent-green-500" - /> + className="w-4 h-4 accent-green-500 cursor-pointer" /> {t.topic}{t.last_payload ?? '-'} - {t.topic}{t.last_payload ?? '—'} + setEditPaths({ ...editPaths, [t.id]: e.target.value })} onBlur={() => handleJsonPathBlur(t.id)} - placeholder="$.temperature" - /> + placeholder="$.value" /> - + +
+ + {/* Pagination */} + {totalPages > 1 && ( +
+ + Seite {page + 1} von {totalPages} + +
+ )}
); } diff --git a/display/middleware/src/api/routes/mqtt-topics.ts b/display/middleware/src/api/routes/mqtt-topics.ts index f99a79b..814e2b9 100644 --- a/display/middleware/src/api/routes/mqtt-topics.ts +++ b/display/middleware/src/api/routes/mqtt-topics.ts @@ -11,11 +11,15 @@ import { getDb } from '../../db/index'; const router = Router({ mergeParams: true }); -// GET /api/mqtt/brokers/:brokerId/topics +// GET /api/mqtt/brokers/:brokerId/topics?search=grid&selected=1&limit=50&offset=0 router.get('/', (req, res): void => { try { const brokerId = parseInt(req.params.brokerId); - res.json(getTopicsForBroker(brokerId)); + const search = req.query.search as string | undefined; + const selectedOnly = req.query.selected === '1'; + const limit = parseInt(req.query.limit as string) || 50; + const offset = parseInt(req.query.offset as string) || 0; + res.json(getTopicsForBroker(brokerId, { search, selectedOnly, limit, offset })); } catch (err) { res.status(500).json({ error: (err as Error).message }); } diff --git a/display/middleware/src/modules/mqtt/topic-discovery.ts b/display/middleware/src/modules/mqtt/topic-discovery.ts index 9ab937a..77a34e4 100644 --- a/display/middleware/src/modules/mqtt/topic-discovery.ts +++ b/display/middleware/src/modules/mqtt/topic-discovery.ts @@ -19,11 +19,32 @@ export interface UpdateTopicData { ignored?: number; } -export function getTopicsForBroker(brokerId: number): MqttTopic[] { +export function getTopicsForBroker(brokerId: number, opts?: { search?: string; selectedOnly?: boolean; limit?: number; offset?: number }): { topics: MqttTopic[]; total: number } { const db = getDb(); - return db - .prepare('SELECT * FROM mqtt_topics WHERE broker_id = ? ORDER BY id ASC') - .all(brokerId) as MqttTopic[]; + const conditions = ['broker_id = ?']; + const params: any[] = [brokerId]; + + if (opts?.search) { + conditions.push('topic LIKE ?'); + params.push(`%${opts.search}%`); + } + if (opts?.selectedOnly) { + conditions.push('selected = 1'); + } + conditions.push('ignored = 0'); + + const where = conditions.join(' AND '); + const total = (db.prepare(`SELECT count(*) as c FROM mqtt_topics WHERE ${where}`).get(...params) as any).c; + + const limit = opts?.limit || 50; + const offset = opts?.offset || 0; + params.push(limit, offset); + + const topics = db + .prepare(`SELECT * FROM mqtt_topics WHERE ${where} ORDER BY selected DESC, last_payload IS NOT NULL DESC, topic ASC LIMIT ? OFFSET ?`) + .all(...params) as MqttTopic[]; + + return { topics, total }; } export function updateTopic(id: number, data: UpdateTopicData): MqttTopic | undefined {