feat: Benchmarks (DB Klartext vs. SQLCipher, Migration) + scripts/test.ps1
benchmark/ misst Sync-Schreiben, App-Start, Chat oeffnen, lokale Suche und die Migrationsdauer (A-B-B-A, Aufwaermlauf, Median), schreibt build/benchmarks/*.json und vergleicht mit benchmark/baseline/. scripts/test.ps1 fuehrt Analyse, Unit-Tests, optional Live-Test (-Live) und Benchmarks (-Bench) mit einem Befehl aus. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Bernd Steckmeister
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
f4d334a60b
commit
df1b392ed8
@@ -0,0 +1,257 @@
|
||||
// Benchmarks sind Test-Code, liegen aber bewusst außerhalb von test/ (sie
|
||||
// sollen in einem normalen `flutter test` NICHT mitlaufen) – deshalb kennt
|
||||
// der Analyzer sie nicht als Tests.
|
||||
// ignore_for_file: invalid_use_of_visible_for_testing_member
|
||||
@TestOn('windows')
|
||||
library;
|
||||
|
||||
import 'dart:io';
|
||||
|
||||
import 'package:flutter_test/flutter_test.dart';
|
||||
import 'package:matrix/matrix.dart';
|
||||
import 'package:path/path.dart' as p;
|
||||
import 'package:pyramid/core/app_database.dart';
|
||||
import 'package:pyramid/features/search/local_message_search.dart';
|
||||
import 'package:sqflite_common_ffi/sqflite_ffi.dart';
|
||||
|
||||
import '../test/support/synthetic_matrix_data.dart';
|
||||
import 'support/bench.dart';
|
||||
|
||||
/// Datenbank-Benchmark: Was kostet die SQLCipher-Verschlüsselung im Alltag?
|
||||
///
|
||||
/// Misst – jeweils für eine Klartext-DB und eine SQLCipher-DB mit DERSELBEN
|
||||
/// Bibliothek aus dem App-Build – die vier Wege, auf denen Nutzer die DB
|
||||
/// spüren:
|
||||
/// 1. Sync-Schreiben (Nachrichten landen in Transaktionen wie beim Sync),
|
||||
/// 2. App-Start (DB öffnen + SDK-Schema + Raumliste laden),
|
||||
/// 3. Chat öffnen (50 neueste Nachrichten eines Raums),
|
||||
/// 4. lokale Suche (Strg+K) – ohne Treffer = Worst Case, scannt alles.
|
||||
///
|
||||
/// Daten sind synthetisch (`test/support/synthetic_matrix_data.dart`), laufen
|
||||
/// aber über die echte SDK-Speicherschicht. Lauf:
|
||||
/// flutter test benchmark/db_benchmark_test.dart
|
||||
/// oder alles zusammen über `scripts/test.ps1 -Bench`.
|
||||
|
||||
const _modes = ['klartext', 'sqlcipher'];
|
||||
|
||||
const _benchKey =
|
||||
'0b1e2c3d4f5a6b7c8d9e0f1a2b3c4d5e6f708192a3b4c5d6e7f8091a2b3c4d5e';
|
||||
|
||||
void main() {
|
||||
final skip = findCipherDll() == null
|
||||
? 'SQLCipher-DLL fehlt – erst "flutter build windows" ausführen'
|
||||
: null;
|
||||
|
||||
test(
|
||||
'DB: Klartext vs. SQLCipher',
|
||||
() async {
|
||||
final rooms = scaled(20);
|
||||
final eventsPerRoom = scaled(500);
|
||||
final totalEvents = rooms * eventsPerRoom;
|
||||
final report = BenchReport(
|
||||
'db',
|
||||
params: {'rooms': rooms, 'eventsPerRoom': eventsPerRoom},
|
||||
);
|
||||
final factory = cipherFactory();
|
||||
final tempDir = await Directory.systemTemp.createTemp('pyramid_bench_db');
|
||||
|
||||
Future<Database> openBenchDb(String mode, String path) async {
|
||||
final db = mode == 'sqlcipher'
|
||||
? await AppDatabase.openEncrypted(factory, path, _benchKey)
|
||||
: await factory.openDatabase(path);
|
||||
await applyAppPragmas(db);
|
||||
return db;
|
||||
}
|
||||
|
||||
try {
|
||||
// Ungemessener Aufwärm-Durchlauf (JIT, DB-Isolate, SQLCipher-Init).
|
||||
// Bewusst mit voller Raumgröße: Der JIT optimiert die SDK-Pfade erst
|
||||
// nach vielen Aufrufen mit wachsender Fragment-Liste – mit einem
|
||||
// Mini-Durchlauf war die erste Messrunde noch ~45 % langsamer.
|
||||
for (final mode in _modes) {
|
||||
final path = p.join(tempDir.path, 'warmup_$mode.sqlite');
|
||||
final client = await SyntheticMatrixData.openClient(
|
||||
await openBenchDb(mode, path),
|
||||
);
|
||||
await SyntheticMatrixData.populate(
|
||||
client,
|
||||
rooms: (rooms ~/ 4).clamp(1, rooms),
|
||||
eventsPerRoom: eventsPerRoom,
|
||||
);
|
||||
await SyntheticMatrixData.loadRooms(client);
|
||||
await localMessageSearch(client, 'aufwärmen');
|
||||
await client.dispose(closeDatabase: true);
|
||||
}
|
||||
|
||||
// Gemessen wird abwechselnd A-B-B-A: Die Schreibkosten sind von
|
||||
// Dart-Code (SDK-JSON) dominiert, und der JIT wird während des Laufs
|
||||
// weiter schneller. Ohne das Muster gewinnt einfach der später
|
||||
// gemessene Modus (2026-10-07 nachgemessen: ±20 % je nach Reihenfolge).
|
||||
const order = ['klartext', 'sqlcipher', 'sqlcipher', 'klartext'];
|
||||
final timings = {
|
||||
for (final mode in _modes) mode: <String, List<double>>{},
|
||||
};
|
||||
void record(String mode, String metric, Iterable<double> values) =>
|
||||
timings[mode]!.putIfAbsent(metric, () => []).addAll(values);
|
||||
|
||||
// 1. Sync-Schreiben – je Runde eine frische DB.
|
||||
final paths = <String, String>{};
|
||||
for (var round = 0; round < order.length; round++) {
|
||||
final mode = order[round];
|
||||
final path = p.join(tempDir.path, '${mode}_$round.sqlite');
|
||||
final client = await SyntheticMatrixData.openClient(
|
||||
await openBenchDb(mode, path),
|
||||
);
|
||||
final watch = Stopwatch()..start();
|
||||
await SyntheticMatrixData.populate(
|
||||
client,
|
||||
rooms: rooms,
|
||||
eventsPerRoom: eventsPerRoom,
|
||||
needleEvery: 97,
|
||||
);
|
||||
watch.stop();
|
||||
await client.dispose(closeDatabase: true);
|
||||
record(mode, 'write', [watch.elapsedMicroseconds / 1000]);
|
||||
paths[mode] = path;
|
||||
}
|
||||
|
||||
// 2. App-Start: DB öffnen + SDK-Schema + Raumliste, danach wieder
|
||||
// schließen – wie ein Neustart. Eigene Runde VOR den Lese-Messungen:
|
||||
// sqflite liefert für denselben Pfad dieselbe Verbindung zurück,
|
||||
// ein Schließen hier würde sonst deren offene Verbindung kappen.
|
||||
for (final mode in order) {
|
||||
record(
|
||||
mode,
|
||||
'startup',
|
||||
await sample(() async {
|
||||
final c = await SyntheticMatrixData.openClient(
|
||||
await openBenchDb(mode, paths[mode]!),
|
||||
);
|
||||
await SyntheticMatrixData.loadRooms(c);
|
||||
await c.dispose(closeDatabase: true);
|
||||
}, runs: 3),
|
||||
);
|
||||
}
|
||||
|
||||
// 3.–4. Lesen auf der jeweils zuletzt geschriebenen DB.
|
||||
final clients = <String, Client>{};
|
||||
final roomLists = <String, List<Room>>{};
|
||||
for (final mode in _modes) {
|
||||
final client = await SyntheticMatrixData.openClient(
|
||||
await openBenchDb(mode, paths[mode]!),
|
||||
);
|
||||
clients[mode] = client;
|
||||
roomLists[mode] = await SyntheticMatrixData.loadRooms(client);
|
||||
expect(
|
||||
roomLists[mode],
|
||||
hasLength(rooms),
|
||||
reason: 'Raumliste vollständig ($mode)',
|
||||
);
|
||||
}
|
||||
for (final mode in order) {
|
||||
final client = clients[mode]!;
|
||||
record(
|
||||
mode,
|
||||
'open_chat',
|
||||
(await sample(() async {
|
||||
for (final room in roomLists[mode]!) {
|
||||
final page = await client.database.getEventList(
|
||||
room,
|
||||
limit: 50,
|
||||
);
|
||||
if (page.length != 50) throw StateError('Seite unvollständig');
|
||||
}
|
||||
}, runs: 3)).map((t) => t / rooms),
|
||||
);
|
||||
record(
|
||||
mode,
|
||||
'search_miss',
|
||||
await sample(
|
||||
() => localMessageSearch(client, 'gibtsnichtwort'),
|
||||
runs: 2,
|
||||
),
|
||||
);
|
||||
record(
|
||||
mode,
|
||||
'search_hit',
|
||||
await sample(
|
||||
() => localMessageSearch(client, 'nadelwort'),
|
||||
runs: 2,
|
||||
),
|
||||
);
|
||||
}
|
||||
for (final client in clients.values) {
|
||||
await client.dispose(closeDatabase: true);
|
||||
}
|
||||
|
||||
for (final mode in _modes) {
|
||||
final t = timings[mode]!;
|
||||
final writeMs = median(t['write']!);
|
||||
report.addTimings(
|
||||
'$mode.write_total_ms',
|
||||
'[$mode] Sync-Schreiben, $totalEvents Nachrichten',
|
||||
t['write']!,
|
||||
);
|
||||
report.addValue(
|
||||
'$mode.write_per_event_us',
|
||||
'[$mode] Sync-Schreiben je Nachricht',
|
||||
writeMs * 1000 / totalEvents,
|
||||
'µs',
|
||||
compare: true,
|
||||
);
|
||||
report.addTimings(
|
||||
'$mode.startup_ms',
|
||||
'[$mode] App-Start: DB + Raumliste',
|
||||
t['startup']!,
|
||||
);
|
||||
report.addTimings(
|
||||
'$mode.open_chat_ms',
|
||||
'[$mode] Chat öffnen (50 Nachr.)',
|
||||
t['open_chat']!,
|
||||
);
|
||||
report.addTimings(
|
||||
'$mode.search_miss_ms',
|
||||
'[$mode] Suche ohne Treffer (Worst Case)',
|
||||
t['search_miss']!,
|
||||
);
|
||||
report.addTimings(
|
||||
'$mode.search_hit_ms',
|
||||
'[$mode] Suche mit Treffern',
|
||||
t['search_hit']!,
|
||||
);
|
||||
report.addValue(
|
||||
'$mode.size_mb',
|
||||
'[$mode] DB-Größe inkl. WAL',
|
||||
dbSizeMb(paths[mode]!),
|
||||
'MB',
|
||||
);
|
||||
}
|
||||
|
||||
for (final (id, label) in [
|
||||
('write_total_ms', 'Sync-Schreiben'),
|
||||
('startup_ms', 'App-Start'),
|
||||
('open_chat_ms', 'Chat öffnen'),
|
||||
('search_miss_ms', 'Suche ohne Treffer'),
|
||||
]) {
|
||||
final plain = report.valueOf('klartext.$id');
|
||||
final enc = report.valueOf('sqlcipher.$id');
|
||||
report.addValue(
|
||||
'cipher_overhead.$id',
|
||||
'Aufschlag Verschlüsselung: $label',
|
||||
(enc / plain - 1) * 100,
|
||||
'%',
|
||||
);
|
||||
}
|
||||
|
||||
final regressions = await report.finish();
|
||||
if (strictBench) expect(regressions, isEmpty);
|
||||
} finally {
|
||||
try {
|
||||
await tempDir.delete(recursive: true);
|
||||
} catch (_) {}
|
||||
}
|
||||
},
|
||||
skip: skip,
|
||||
timeout: const Timeout(Duration(minutes: 20)),
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user