diff --git a/docs/POSTGRES_SETUP.md b/docs/POSTGRES_SETUP.md index aca13e6..d661710 100644 --- a/docs/POSTGRES_SETUP.md +++ b/docs/POSTGRES_SETUP.md @@ -1,284 +1,349 @@ # PostgreSQL + TimescaleDB Setup -Anleitung fuer die Einrichtung von PostgreSQL mit TimescaleDB-Extension auf dem RMM-Backend-Server (your-backend, Debian 13 Trixie). +Einrichtung von PostgreSQL mit TimescaleDB auf dem RMM-Backend-Server (Debian 13 Trixie). + +> **Hinweis:** Das Backend legt alle Tabellen beim ersten Start **automatisch an** (Auto-Migration inkl. ALTER TABLE für neue Spalten). Das Schema muss **nicht** manuell erstellt werden. Diese Datei dient als Referenz und Installationsanleitung. + +--- ## 1. PostgreSQL installieren ```bash -# Debian-Pakete apt update apt install -y postgresql postgresql-contrib postgresql-common -# Dienst starten und aktivieren systemctl enable postgresql systemctl start postgresql -# Version pruefen +# Version prüfen (Trixie liefert PG17) psql --version -# Erwartet: psql (PostgreSQL) 17.x (Debian 13 Trixie liefert PG17) ``` +--- + ## 2. TimescaleDB installieren ```bash -# TimescaleDB APT-Repository hinzufuegen +# Repository einrichten apt install -y curl gnupg -curl -s https://packagecloud.io/install/repositories/timescale/timescaledb/script.deb.sh | bash - -# Bookworm-Repo verwenden (Trixie-Repo existiert noch nicht) echo "deb https://packagecloud.io/timescale/timescaledb/debian/ bookworm main" \ > /etc/apt/sources.list.d/timescaledb.list -curl -sL https://packagecloud.io/timescale/timescaledb/gpgkey | gpg --dearmor > /etc/apt/trusted.gpg.d/timescaledb.gpg +curl -sL https://packagecloud.io/timescale/timescaledb/gpgkey | \ + gpg --dearmor > /etc/apt/trusted.gpg.d/timescaledb.gpg apt update -# WICHTIG: Version muss zur installierten PG-Version passen -# Pruefen: dpkg -l | grep postgresql-17 → z.B. 17.8 -# Dann passende TimescaleDB-Version installieren: +# Passende Version zu PG17 installieren apt install -y timescaledb-2-loader-postgresql-17=2.25.1~debian12-1708 \ timescaledb-2-postgresql-17=2.25.1~debian12-1708 -# shared_preload_libraries setzen +# In postgresql.conf aktivieren sed -i "s/#shared_preload_libraries = ''/shared_preload_libraries = 'timescaledb'/" \ /etc/postgresql/17/main/postgresql.conf -# PostgreSQL neustarten systemctl restart postgresql ``` +--- + ## 3. Datenbank und User anlegen ```bash -# Als postgres User -sudo -u postgres psql -``` - -```sql --- Datenbank anlegen +sudo -u postgres psql << 'EOF' CREATE DATABASE rmm; - --- User anlegen (Passwort anpassen!) -CREATE USER rmm WITH PASSWORD 'YOUR_DB_PASSWORD'; - --- Rechte vergeben +CREATE USER rmm WITH PASSWORD 'SICHERES_PASSWORT'; GRANT ALL PRIVILEGES ON DATABASE rmm TO rmm; - --- In die rmm-Datenbank wechseln \c rmm - --- Schema-Rechte GRANT ALL ON SCHEMA public TO rmm; - --- TimescaleDB Extension aktivieren CREATE EXTENSION IF NOT EXISTS timescaledb; - --- Pruefen \dx --- Sollte timescaledb in der Liste zeigen - -\q +EOF ``` -## 4. pg_hba.conf anpassen (lokaler Zugriff) +--- + +## 4. pg_hba.conf anpassen ```bash -# Zeile hinzufuegen (lokaler TCP-Zugriff fuer rmm User) echo "host rmm rmm 127.0.0.1/32 scram-sha-256" >> /etc/postgresql/17/main/pg_hba.conf - -# PostgreSQL neu laden systemctl reload postgresql ``` +--- + ## 5. Verbindung testen ```bash -psql -h 127.0.0.1 -U rmm -d rmm -c "SELECT version();" -# Passwort: YOUR_DB_PASSWORD - psql -h 127.0.0.1 -U rmm -d rmm -c "SELECT extname, extversion FROM pg_extension WHERE extname = 'timescaledb';" -# Sollte timescaledb mit Version zeigen ``` -## 6. Schema anlegen +--- -Das Backend legt die Tabellen automatisch beim Start an (Auto-Migration). Die Struktur: +## 6. Schema (Referenz — wird automatisch angelegt) -### Relationale Tabellen +### `agents` — Agent-Registry ```sql --- Agent-Registry -CREATE TABLE IF NOT EXISTS agents ( - id TEXT PRIMARY KEY, - name TEXT NOT NULL, - hostname TEXT NOT NULL, - ip TEXT NOT NULL, - opnsense_version TEXT NOT NULL DEFAULT '', - registered_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - last_heartbeat TIMESTAMPTZ +CREATE TABLE agents ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + hostname TEXT NOT NULL, + ip TEXT NOT NULL, + opnsense_version TEXT NOT NULL DEFAULT '', + agent_version TEXT NOT NULL DEFAULT '', + platform TEXT NOT NULL DEFAULT 'freebsd', + registered_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + last_heartbeat TIMESTAMPTZ, + customer_id INTEGER REFERENCES customers(id), + update_requested BOOLEAN NOT NULL DEFAULT FALSE ); +CREATE INDEX idx_agents_customer ON agents(customer_id); +``` --- Config-Backups (versioniert, dedupliziert nach Hash) -CREATE TABLE IF NOT EXISTS config_backups ( - id SERIAL PRIMARY KEY, - agent_id TEXT NOT NULL REFERENCES agents(id) ON DELETE CASCADE, - hash TEXT NOT NULL, - size INTEGER NOT NULL, - config_xml TEXT NOT NULL, +### `customers` — Kundenstammdaten + +```sql +CREATE TABLE customers ( + id SERIAL PRIMARY KEY, + number TEXT NOT NULL UNIQUE, -- z.B. "K05068" + name TEXT NOT NULL, created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); - -CREATE INDEX IF NOT EXISTS idx_config_backups_agent ON config_backups(agent_id, created_at DESC); -CREATE INDEX IF NOT EXISTS idx_config_backups_hash ON config_backups(agent_id, hash); ``` -### TimescaleDB Hypertable (Metriken) +### `users` — Frontend-Benutzer ```sql --- Metriken-Tabelle -CREATE TABLE IF NOT EXISTS metrics ( - time TIMESTAMPTZ NOT NULL, - agent_id TEXT NOT NULL, - metric TEXT NOT NULL, -- z.B. 'cpu_usage', 'memory_used_percent', 'disk_used_percent' - value DOUBLE PRECISION NOT NULL, - tags JSONB -- z.B. {"interface":"igb0"}, {"disk":"/dev/ada0"} +CREATE TABLE users ( + id SERIAL PRIMARY KEY, + username TEXT NOT NULL UNIQUE, + password_hash TEXT NOT NULL, -- bcrypt + display_name TEXT NOT NULL DEFAULT '', + role TEXT NOT NULL DEFAULT 'admin', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + last_login TIMESTAMPTZ ); - --- In Hypertable umwandeln (automatische Partitionierung nach Zeit) -SELECT create_hypertable('metrics', 'time', if_not_exists => TRUE); - --- Index fuer schnelle Agent+Metrik-Abfragen -CREATE INDEX IF NOT EXISTS idx_metrics_agent_metric ON metrics(agent_id, metric, time DESC); +-- Standard-Admin: admin / Start!123 (beim ersten Backend-Start angelegt) ``` -### Systemdaten-Snapshots (aktueller Zustand) +### `system_data` — Aktueller Systemzustand pro Agent ```sql --- Aktueller Snapshot der vollstaendigen Systemdaten (JSON) -CREATE TABLE IF NOT EXISTS system_data ( - agent_id TEXT PRIMARY KEY REFERENCES agents(id) ON DELETE CASCADE, - data_json JSONB NOT NULL, +CREATE TABLE system_data ( + agent_id TEXT PRIMARY KEY REFERENCES agents(id) ON DELETE CASCADE, + data_json JSONB NOT NULL, updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); ``` -## 7. Retention Policies einrichten +### `config_backups` — Versionierte OPNsense-Configs ```sql --- Metriken aelter als 90 Tage automatisch loeschen -SELECT add_retention_policy('metrics', INTERVAL '90 days'); - --- Optional: Continuous Aggregate fuer stuendliche Durchschnitte -CREATE MATERIALIZED VIEW metrics_hourly -WITH (timescaledb.continuous) AS -SELECT - time_bucket('1 hour', time) AS bucket, - agent_id, - metric, - AVG(value) AS avg_value, - MIN(value) AS min_value, - MAX(value) AS max_value, - COUNT(*) AS sample_count -FROM metrics -GROUP BY bucket, agent_id, metric -WITH NO DATA; - --- Automatisch aktualisieren -SELECT add_continuous_aggregate_policy('metrics_hourly', - start_offset => INTERVAL '3 hours', - end_offset => INTERVAL '1 hour', - schedule_interval => INTERVAL '1 hour' +CREATE TABLE config_backups ( + id SERIAL PRIMARY KEY, + agent_id TEXT NOT NULL REFERENCES agents(id) ON DELETE CASCADE, + hash TEXT NOT NULL, -- SHA256 (Deduplizierung) + size INTEGER NOT NULL, + config_xml TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); - --- Stuendliche Aggregate 1 Jahr behalten -SELECT add_retention_policy('metrics_hourly', INTERVAL '365 days'); +CREATE INDEX idx_config_backups_agent ON config_backups(agent_id, created_at DESC); +CREATE INDEX idx_config_backups_hash ON config_backups(agent_id, hash); ``` -## 8. Compression aktivieren (optional, spart Speicher) +### `agent_events` — Online/Offline-Ereignisse ```sql --- Compression fuer Metriken aelter als 7 Tage +CREATE TABLE agent_events ( + id SERIAL PRIMARY KEY, + agent_id TEXT NOT NULL REFERENCES agents(id) ON DELETE CASCADE, + event_type TEXT NOT NULL, -- "online","offline","stale","connected","disconnected" + message TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); +CREATE INDEX idx_agent_events_agent ON agent_events(agent_id, created_at DESC); +``` + +### `agent_firmware` — Agent-Binaries + +```sql +CREATE TABLE agent_firmware ( + version TEXT NOT NULL, + platform TEXT NOT NULL DEFAULT 'freebsd', -- "freebsd","linux","windows" + hash TEXT NOT NULL, -- SHA256 + binary_data BYTEA NOT NULL, + uploaded_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE (version, platform) +); +``` + +### `tasks` — Task-Scheduler + +```sql +CREATE TABLE tasks ( + id SERIAL PRIMARY KEY, + agent_id TEXT NOT NULL REFERENCES agents(id) ON DELETE CASCADE, + name TEXT DEFAULT '', + action TEXT NOT NULL, -- "update","backup","reboot",... + status TEXT NOT NULL DEFAULT 'scheduled', + schedule_type TEXT DEFAULT 'once', -- "once","daily","weekly","monthly" + schedule_time TEXT DEFAULT '02:00', + schedule_weekday INT, -- 0=So..6=Sa + schedule_monthday INT, -- 1-31 + reboot BOOLEAN DEFAULT FALSE, + health_check BOOLEAN DEFAULT FALSE, + health_check_timeout INT DEFAULT 600, + active BOOLEAN DEFAULT TRUE, + scheduled_at TIMESTAMPTZ NOT NULL, + started_at TIMESTAMPTZ, + finished_at TIMESTAMPTZ, + last_run TIMESTAMPTZ, + next_run TIMESTAMPTZ, + run_count INT DEFAULT 0, + result JSONB, + webhook_url TEXT, + webhook_sent BOOLEAN DEFAULT FALSE, + webhook_response TEXT, + created_by TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); +CREATE INDEX idx_tasks_status_scheduled ON tasks(status, scheduled_at); +CREATE INDEX idx_tasks_active_next_run ON tasks(active, next_run) WHERE active = TRUE; +``` + +### `api_keys` — API-Keys + +```sql +CREATE TABLE api_keys ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + key TEXT NOT NULL UNIQUE, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + last_used TIMESTAMPTZ +); +``` + +### `installers` — OPNsense Plugin-Pakete + +```sql +CREATE TABLE installers ( + id SERIAL PRIMARY KEY, + platform TEXT NOT NULL UNIQUE, + filename TEXT NOT NULL, + data BYTEA NOT NULL, + sha256 TEXT NOT NULL, + size INTEGER NOT NULL, + uploaded_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); +``` + +### `audit_log` — Audit-Protokoll + +```sql +CREATE TABLE audit_log ( + id SERIAL PRIMARY KEY, + timestamp TIMESTAMPTZ NOT NULL DEFAULT NOW(), + user_name TEXT NOT NULL, + action TEXT NOT NULL, + target_type TEXT, + target_id TEXT, + target_name TEXT, + details TEXT, + source_ip TEXT +); +CREATE INDEX idx_audit_log_timestamp ON audit_log(timestamp DESC); +``` + +### `system_settings` — Backend-Einstellungen + +```sql +CREATE TABLE system_settings ( + key TEXT PRIMARY KEY, -- "backend_url_internal","backend_url_external","api_key" + value TEXT NOT NULL, + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); +``` + +### `metrics` — TimescaleDB Hypertable + +```sql +CREATE TABLE metrics ( + time TIMESTAMPTZ NOT NULL, + agent_id TEXT NOT NULL, + metric TEXT NOT NULL, + value DOUBLE PRECISION NOT NULL, + tags JSONB +); +SELECT create_hypertable('metrics', 'time', if_not_exists => TRUE); +CREATE INDEX idx_metrics_agent_metric ON metrics(agent_id, metric, time DESC); + +-- Retention: Rohdaten nach 90 Tagen löschen +SELECT add_retention_policy('metrics', INTERVAL '90 days'); + +-- Compression: Daten älter als 7 Tage komprimieren ALTER TABLE metrics SET ( timescaledb.compress, timescaledb.compress_segmentby = 'agent_id, metric', timescaledb.compress_orderby = 'time DESC' ); - SELECT add_compression_policy('metrics', INTERVAL '7 days'); ``` -## 9. Backend-Konfiguration +--- -In der `config.yaml` des Backends: - -```yaml -# Alte SQLite-Config entfernen: -# db_path: "rmm.db" - -# Neue PostgreSQL-Config: -database: - host: "127.0.0.1" - port: 5432 - user: "rmm" - password: "YOUR_DB_PASSWORD" - dbname: "rmm" - sslmode: "disable" -``` - -## 10. Metriken die der Agent sendet - -Bei jedem Heartbeat (alle 60s) werden folgende Metriken geschrieben: - -| Metrik | Beschreibung | Tags | -|--------|-------------|------| -| `cpu_usage` | CPU-Auslastung in Prozent | — | -| `memory_used_percent` | RAM-Nutzung in Prozent | — | -| `memory_used_bytes` | RAM genutzt in Bytes | — | -| `memory_total_bytes` | RAM gesamt in Bytes | — | -| `disk_used_percent` | Disk-Belegung pro Mountpoint | `{"mountpoint":"/", "dataset":"zroot/ROOT"}` | -| `uptime_seconds` | Uptime in Sekunden | — | -| `interface_rx_bytes` | Empfangene Bytes pro Interface | `{"interface":"igb0"}` | -| `interface_tx_bytes` | Gesendete Bytes pro Interface | `{"interface":"igb0"}` | -| `gateway_rtt_ms` | Gateway RTT in ms | `{"gateway":"WAN_GW"}` | -| `gateway_loss_percent` | Gateway Paketverlust | `{"gateway":"WAN_GW"}` | - -## Nuetzliche Abfragen +## 7. Nützliche Abfragen ```sql --- CPU-Auslastung der letzten 24h (5-Minuten-Durchschnitt) -SELECT time_bucket('5 minutes', time) AS t, - agent_id, - AVG(value) AS cpu_avg -FROM metrics -WHERE metric = 'cpu_usage' - AND time > NOW() - INTERVAL '24 hours' -GROUP BY t, agent_id -ORDER BY t; +-- Alle Agents mit Status +SELECT id, name, hostname, ip, opnsense_version, agent_version, + last_heartbeat, + CASE + WHEN last_heartbeat > NOW() - INTERVAL '2 min' THEN 'online' + WHEN last_heartbeat > NOW() - INTERVAL '5 min' THEN 'stale' + ELSE 'offline' + END AS status +FROM agents ORDER BY name; --- Aktuelle Werte aller Agents -SELECT DISTINCT ON (agent_id, metric) - agent_id, metric, value, time +-- CPU-Auslastung letzte 24h (5-Minuten-Durchschnitt) +SELECT time_bucket('5 minutes', time) AS t, agent_id, AVG(value) AS cpu_avg FROM metrics -ORDER BY agent_id, metric, time DESC; +WHERE metric = 'cpu_usage' AND time > NOW() - INTERVAL '24 hours' +GROUP BY t, agent_id ORDER BY t; -- Disk-Warnung: > 80% belegt -SELECT agent_id, tags->>'mountpoint' AS mount, - value AS used_percent, time +SELECT agent_id, tags->>'mountpoint' AS mount, value AS used_pct, time FROM metrics -WHERE metric = 'disk_used_percent' - AND value > 80 +WHERE metric = 'disk_used_percent' AND value > 80 AND time > NOW() - INTERVAL '1 hour' ORDER BY value DESC; + +-- Offene Tasks +SELECT t.id, a.name AS agent, t.action, t.status, t.scheduled_at, t.webhook_url +FROM tasks t JOIN agents a ON t.agent_id = a.id +WHERE t.status IN ('scheduled','running') +ORDER BY t.scheduled_at; + +-- Datenbankgrösse +SELECT pg_size_pretty(pg_database_size('rmm')) AS db_size; +SELECT relname AS tabelle, pg_size_pretty(pg_total_relation_size(relid)) AS groesse +FROM pg_catalog.pg_statio_user_tables ORDER BY pg_total_relation_size(relid) DESC; ``` -## Zusammenfassung +--- -| Komponente | Zweck | -|-----------|-------| -| PostgreSQL 17 | Relationale Daten (Agents, Backups) | -| TimescaleDB 2.x | Time-Series Metriken (CPU, RAM, Disk, ...) | -| `agents` | Agent-Registry | -| `config_backups` | Versionierte Firewall-Configs | -| `system_data` | Aktueller JSON-Snapshot | -| `metrics` | Hypertable mit 60s-Aufloesung | -| `metrics_hourly` | Continuous Aggregate (1h) | -| Retention | Raw: 90 Tage, Hourly: 365 Tage | -| Compression | Nach 7 Tagen automatisch | +## Übersicht + +| Tabelle | Typ | Beschreibung | +|---------|-----|-------------| +| `agents` | Relational | Agent-Registry | +| `customers` | Relational | Kundenstammdaten | +| `users` | Relational | Frontend-Benutzer | +| `system_data` | Relational | Aktueller Systemzustand (JSONB) | +| `config_backups` | Relational | Versionierte OPNsense-Configs | +| `agent_events` | Relational | Online/Offline-Ereignisse | +| `agent_firmware` | Relational | Agent-Binaries pro Plattform | +| `tasks` | Relational | Task-Scheduler mit Webhook | +| `api_keys` | Relational | API-Keys | +| `installers` | Relational | OPNsense Plugin-Pakete | +| `audit_log` | Relational | Audit-Protokoll | +| `system_settings` | Relational | Backend-Einstellungen (Key/Value) | +| `metrics` | TimescaleDB | Zeitreihen (CPU, RAM, Disk, Network, Gateway) |