feat: Benchmarks (DB Klartext vs. SQLCipher, Migration) + scripts/test.ps1
benchmark/ misst Sync-Schreiben, App-Start, Chat oeffnen, lokale Suche und die Migrationsdauer (A-B-B-A, Aufwaermlauf, Median), schreibt build/benchmarks/*.json und vergleicht mit benchmark/baseline/. scripts/test.ps1 fuehrt Analyse, Unit-Tests, optional Live-Test (-Live) und Benchmarks (-Bench) mit einem Befehl aus. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Bernd Steckmeister
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
f4d334a60b
commit
df1b392ed8
@@ -0,0 +1,151 @@
|
||||
{
|
||||
"suite": "db",
|
||||
"timestamp": "2026-10-07T11:35:13.322832",
|
||||
"git": "b2413ef",
|
||||
"host": {
|
||||
"os": "\"Windows 11 Home\" 10.0 (Build 26200)",
|
||||
"cpus": 12
|
||||
},
|
||||
"params": {
|
||||
"rooms": 20,
|
||||
"eventsPerRoom": 500
|
||||
},
|
||||
"metrics": {
|
||||
"klartext.write_total_ms": {
|
||||
"label": "[klartext] Sync-Schreiben, 10000 Nachrichten",
|
||||
"unit": "ms",
|
||||
"value": 2513.443,
|
||||
"min": 2469.054,
|
||||
"runs": 2,
|
||||
"compare": true
|
||||
},
|
||||
"klartext.write_per_event_us": {
|
||||
"label": "[klartext] Sync-Schreiben je Nachricht",
|
||||
"unit": "µs",
|
||||
"value": 251.344,
|
||||
"runs": 1,
|
||||
"compare": true
|
||||
},
|
||||
"klartext.startup_ms": {
|
||||
"label": "[klartext] App-Start: DB + Raumliste",
|
||||
"unit": "ms",
|
||||
"value": 4.121,
|
||||
"min": 3.254,
|
||||
"runs": 6,
|
||||
"compare": true
|
||||
},
|
||||
"klartext.open_chat_ms": {
|
||||
"label": "[klartext] Chat öffnen (50 Nachr.)",
|
||||
"unit": "ms",
|
||||
"value": 0.328,
|
||||
"min": 0.286,
|
||||
"runs": 6,
|
||||
"compare": true
|
||||
},
|
||||
"klartext.search_miss_ms": {
|
||||
"label": "[klartext] Suche ohne Treffer (Worst Case)",
|
||||
"unit": "ms",
|
||||
"value": 63.897,
|
||||
"min": 63.716,
|
||||
"runs": 4,
|
||||
"compare": true
|
||||
},
|
||||
"klartext.search_hit_ms": {
|
||||
"label": "[klartext] Suche mit Treffern",
|
||||
"unit": "ms",
|
||||
"value": 31.777,
|
||||
"min": 31.202,
|
||||
"runs": 4,
|
||||
"compare": true
|
||||
},
|
||||
"klartext.size_mb": {
|
||||
"label": "[klartext] DB-Größe inkl. WAL",
|
||||
"unit": "MB",
|
||||
"value": 4.008,
|
||||
"runs": 1,
|
||||
"compare": false
|
||||
},
|
||||
"sqlcipher.write_total_ms": {
|
||||
"label": "[sqlcipher] Sync-Schreiben, 10000 Nachrichten",
|
||||
"unit": "ms",
|
||||
"value": 2615.633,
|
||||
"min": 2456.356,
|
||||
"runs": 2,
|
||||
"compare": true
|
||||
},
|
||||
"sqlcipher.write_per_event_us": {
|
||||
"label": "[sqlcipher] Sync-Schreiben je Nachricht",
|
||||
"unit": "µs",
|
||||
"value": 261.563,
|
||||
"runs": 1,
|
||||
"compare": true
|
||||
},
|
||||
"sqlcipher.startup_ms": {
|
||||
"label": "[sqlcipher] App-Start: DB + Raumliste",
|
||||
"unit": "ms",
|
||||
"value": 4.313,
|
||||
"min": 3.718,
|
||||
"runs": 6,
|
||||
"compare": true
|
||||
},
|
||||
"sqlcipher.open_chat_ms": {
|
||||
"label": "[sqlcipher] Chat öffnen (50 Nachr.)",
|
||||
"unit": "ms",
|
||||
"value": 0.323,
|
||||
"min": 0.309,
|
||||
"runs": 6,
|
||||
"compare": true
|
||||
},
|
||||
"sqlcipher.search_miss_ms": {
|
||||
"label": "[sqlcipher] Suche ohne Treffer (Worst Case)",
|
||||
"unit": "ms",
|
||||
"value": 63.748,
|
||||
"min": 62.8,
|
||||
"runs": 4,
|
||||
"compare": true
|
||||
},
|
||||
"sqlcipher.search_hit_ms": {
|
||||
"label": "[sqlcipher] Suche mit Treffern",
|
||||
"unit": "ms",
|
||||
"value": 31.676,
|
||||
"min": 31.296,
|
||||
"runs": 4,
|
||||
"compare": true
|
||||
},
|
||||
"sqlcipher.size_mb": {
|
||||
"label": "[sqlcipher] DB-Größe inkl. WAL",
|
||||
"unit": "MB",
|
||||
"value": 4.098,
|
||||
"runs": 1,
|
||||
"compare": false
|
||||
},
|
||||
"cipher_overhead.write_total_ms": {
|
||||
"label": "Aufschlag Verschlüsselung: Sync-Schreiben",
|
||||
"unit": "%",
|
||||
"value": 4.066,
|
||||
"runs": 1,
|
||||
"compare": false
|
||||
},
|
||||
"cipher_overhead.startup_ms": {
|
||||
"label": "Aufschlag Verschlüsselung: App-Start",
|
||||
"unit": "%",
|
||||
"value": 4.634,
|
||||
"runs": 1,
|
||||
"compare": false
|
||||
},
|
||||
"cipher_overhead.open_chat_ms": {
|
||||
"label": "Aufschlag Verschlüsselung: Chat öffnen",
|
||||
"unit": "%",
|
||||
"value": -1.388,
|
||||
"runs": 1,
|
||||
"compare": false
|
||||
},
|
||||
"cipher_overhead.search_miss_ms": {
|
||||
"label": "Aufschlag Verschlüsselung: Suche ohne Treffer",
|
||||
"unit": "%",
|
||||
"value": -0.233,
|
||||
"runs": 1,
|
||||
"compare": false
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
{
|
||||
"suite": "migration",
|
||||
"timestamp": "2026-10-07T11:35:21.106818",
|
||||
"git": "b2413ef",
|
||||
"host": {
|
||||
"os": "\"Windows 11 Home\" 10.0 (Build 26200)",
|
||||
"cpus": 12
|
||||
},
|
||||
"params": {
|
||||
"rooms": 20,
|
||||
"eventsPerRoom": 1000
|
||||
},
|
||||
"metrics": {
|
||||
"db_size_mb": {
|
||||
"label": "DB-Größe vor Migration",
|
||||
"unit": "MB",
|
||||
"value": 7.793,
|
||||
"runs": 1,
|
||||
"compare": false
|
||||
},
|
||||
"migrate_ms": {
|
||||
"label": "Migration gesamt",
|
||||
"unit": "ms",
|
||||
"value": 325.058,
|
||||
"min": 322.058,
|
||||
"runs": 3,
|
||||
"compare": true
|
||||
},
|
||||
"migrate_ms_per_mb": {
|
||||
"label": "Migration je MB",
|
||||
"unit": "ms/MB",
|
||||
"value": 41.712,
|
||||
"runs": 1,
|
||||
"compare": true
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,257 @@
|
||||
// Benchmarks sind Test-Code, liegen aber bewusst außerhalb von test/ (sie
|
||||
// sollen in einem normalen `flutter test` NICHT mitlaufen) – deshalb kennt
|
||||
// der Analyzer sie nicht als Tests.
|
||||
// ignore_for_file: invalid_use_of_visible_for_testing_member
|
||||
@TestOn('windows')
|
||||
library;
|
||||
|
||||
import 'dart:io';
|
||||
|
||||
import 'package:flutter_test/flutter_test.dart';
|
||||
import 'package:matrix/matrix.dart';
|
||||
import 'package:path/path.dart' as p;
|
||||
import 'package:pyramid/core/app_database.dart';
|
||||
import 'package:pyramid/features/search/local_message_search.dart';
|
||||
import 'package:sqflite_common_ffi/sqflite_ffi.dart';
|
||||
|
||||
import '../test/support/synthetic_matrix_data.dart';
|
||||
import 'support/bench.dart';
|
||||
|
||||
/// Datenbank-Benchmark: Was kostet die SQLCipher-Verschlüsselung im Alltag?
|
||||
///
|
||||
/// Misst – jeweils für eine Klartext-DB und eine SQLCipher-DB mit DERSELBEN
|
||||
/// Bibliothek aus dem App-Build – die vier Wege, auf denen Nutzer die DB
|
||||
/// spüren:
|
||||
/// 1. Sync-Schreiben (Nachrichten landen in Transaktionen wie beim Sync),
|
||||
/// 2. App-Start (DB öffnen + SDK-Schema + Raumliste laden),
|
||||
/// 3. Chat öffnen (50 neueste Nachrichten eines Raums),
|
||||
/// 4. lokale Suche (Strg+K) – ohne Treffer = Worst Case, scannt alles.
|
||||
///
|
||||
/// Daten sind synthetisch (`test/support/synthetic_matrix_data.dart`), laufen
|
||||
/// aber über die echte SDK-Speicherschicht. Lauf:
|
||||
/// flutter test benchmark/db_benchmark_test.dart
|
||||
/// oder alles zusammen über `scripts/test.ps1 -Bench`.
|
||||
|
||||
const _modes = ['klartext', 'sqlcipher'];
|
||||
|
||||
const _benchKey =
|
||||
'0b1e2c3d4f5a6b7c8d9e0f1a2b3c4d5e6f708192a3b4c5d6e7f8091a2b3c4d5e';
|
||||
|
||||
void main() {
|
||||
final skip = findCipherDll() == null
|
||||
? 'SQLCipher-DLL fehlt – erst "flutter build windows" ausführen'
|
||||
: null;
|
||||
|
||||
test(
|
||||
'DB: Klartext vs. SQLCipher',
|
||||
() async {
|
||||
final rooms = scaled(20);
|
||||
final eventsPerRoom = scaled(500);
|
||||
final totalEvents = rooms * eventsPerRoom;
|
||||
final report = BenchReport(
|
||||
'db',
|
||||
params: {'rooms': rooms, 'eventsPerRoom': eventsPerRoom},
|
||||
);
|
||||
final factory = cipherFactory();
|
||||
final tempDir = await Directory.systemTemp.createTemp('pyramid_bench_db');
|
||||
|
||||
Future<Database> openBenchDb(String mode, String path) async {
|
||||
final db = mode == 'sqlcipher'
|
||||
? await AppDatabase.openEncrypted(factory, path, _benchKey)
|
||||
: await factory.openDatabase(path);
|
||||
await applyAppPragmas(db);
|
||||
return db;
|
||||
}
|
||||
|
||||
try {
|
||||
// Ungemessener Aufwärm-Durchlauf (JIT, DB-Isolate, SQLCipher-Init).
|
||||
// Bewusst mit voller Raumgröße: Der JIT optimiert die SDK-Pfade erst
|
||||
// nach vielen Aufrufen mit wachsender Fragment-Liste – mit einem
|
||||
// Mini-Durchlauf war die erste Messrunde noch ~45 % langsamer.
|
||||
for (final mode in _modes) {
|
||||
final path = p.join(tempDir.path, 'warmup_$mode.sqlite');
|
||||
final client = await SyntheticMatrixData.openClient(
|
||||
await openBenchDb(mode, path),
|
||||
);
|
||||
await SyntheticMatrixData.populate(
|
||||
client,
|
||||
rooms: (rooms ~/ 4).clamp(1, rooms),
|
||||
eventsPerRoom: eventsPerRoom,
|
||||
);
|
||||
await SyntheticMatrixData.loadRooms(client);
|
||||
await localMessageSearch(client, 'aufwärmen');
|
||||
await client.dispose(closeDatabase: true);
|
||||
}
|
||||
|
||||
// Gemessen wird abwechselnd A-B-B-A: Die Schreibkosten sind von
|
||||
// Dart-Code (SDK-JSON) dominiert, und der JIT wird während des Laufs
|
||||
// weiter schneller. Ohne das Muster gewinnt einfach der später
|
||||
// gemessene Modus (2026-10-07 nachgemessen: ±20 % je nach Reihenfolge).
|
||||
const order = ['klartext', 'sqlcipher', 'sqlcipher', 'klartext'];
|
||||
final timings = {
|
||||
for (final mode in _modes) mode: <String, List<double>>{},
|
||||
};
|
||||
void record(String mode, String metric, Iterable<double> values) =>
|
||||
timings[mode]!.putIfAbsent(metric, () => []).addAll(values);
|
||||
|
||||
// 1. Sync-Schreiben – je Runde eine frische DB.
|
||||
final paths = <String, String>{};
|
||||
for (var round = 0; round < order.length; round++) {
|
||||
final mode = order[round];
|
||||
final path = p.join(tempDir.path, '${mode}_$round.sqlite');
|
||||
final client = await SyntheticMatrixData.openClient(
|
||||
await openBenchDb(mode, path),
|
||||
);
|
||||
final watch = Stopwatch()..start();
|
||||
await SyntheticMatrixData.populate(
|
||||
client,
|
||||
rooms: rooms,
|
||||
eventsPerRoom: eventsPerRoom,
|
||||
needleEvery: 97,
|
||||
);
|
||||
watch.stop();
|
||||
await client.dispose(closeDatabase: true);
|
||||
record(mode, 'write', [watch.elapsedMicroseconds / 1000]);
|
||||
paths[mode] = path;
|
||||
}
|
||||
|
||||
// 2. App-Start: DB öffnen + SDK-Schema + Raumliste, danach wieder
|
||||
// schließen – wie ein Neustart. Eigene Runde VOR den Lese-Messungen:
|
||||
// sqflite liefert für denselben Pfad dieselbe Verbindung zurück,
|
||||
// ein Schließen hier würde sonst deren offene Verbindung kappen.
|
||||
for (final mode in order) {
|
||||
record(
|
||||
mode,
|
||||
'startup',
|
||||
await sample(() async {
|
||||
final c = await SyntheticMatrixData.openClient(
|
||||
await openBenchDb(mode, paths[mode]!),
|
||||
);
|
||||
await SyntheticMatrixData.loadRooms(c);
|
||||
await c.dispose(closeDatabase: true);
|
||||
}, runs: 3),
|
||||
);
|
||||
}
|
||||
|
||||
// 3.–4. Lesen auf der jeweils zuletzt geschriebenen DB.
|
||||
final clients = <String, Client>{};
|
||||
final roomLists = <String, List<Room>>{};
|
||||
for (final mode in _modes) {
|
||||
final client = await SyntheticMatrixData.openClient(
|
||||
await openBenchDb(mode, paths[mode]!),
|
||||
);
|
||||
clients[mode] = client;
|
||||
roomLists[mode] = await SyntheticMatrixData.loadRooms(client);
|
||||
expect(
|
||||
roomLists[mode],
|
||||
hasLength(rooms),
|
||||
reason: 'Raumliste vollständig ($mode)',
|
||||
);
|
||||
}
|
||||
for (final mode in order) {
|
||||
final client = clients[mode]!;
|
||||
record(
|
||||
mode,
|
||||
'open_chat',
|
||||
(await sample(() async {
|
||||
for (final room in roomLists[mode]!) {
|
||||
final page = await client.database.getEventList(
|
||||
room,
|
||||
limit: 50,
|
||||
);
|
||||
if (page.length != 50) throw StateError('Seite unvollständig');
|
||||
}
|
||||
}, runs: 3)).map((t) => t / rooms),
|
||||
);
|
||||
record(
|
||||
mode,
|
||||
'search_miss',
|
||||
await sample(
|
||||
() => localMessageSearch(client, 'gibtsnichtwort'),
|
||||
runs: 2,
|
||||
),
|
||||
);
|
||||
record(
|
||||
mode,
|
||||
'search_hit',
|
||||
await sample(
|
||||
() => localMessageSearch(client, 'nadelwort'),
|
||||
runs: 2,
|
||||
),
|
||||
);
|
||||
}
|
||||
for (final client in clients.values) {
|
||||
await client.dispose(closeDatabase: true);
|
||||
}
|
||||
|
||||
for (final mode in _modes) {
|
||||
final t = timings[mode]!;
|
||||
final writeMs = median(t['write']!);
|
||||
report.addTimings(
|
||||
'$mode.write_total_ms',
|
||||
'[$mode] Sync-Schreiben, $totalEvents Nachrichten',
|
||||
t['write']!,
|
||||
);
|
||||
report.addValue(
|
||||
'$mode.write_per_event_us',
|
||||
'[$mode] Sync-Schreiben je Nachricht',
|
||||
writeMs * 1000 / totalEvents,
|
||||
'µs',
|
||||
compare: true,
|
||||
);
|
||||
report.addTimings(
|
||||
'$mode.startup_ms',
|
||||
'[$mode] App-Start: DB + Raumliste',
|
||||
t['startup']!,
|
||||
);
|
||||
report.addTimings(
|
||||
'$mode.open_chat_ms',
|
||||
'[$mode] Chat öffnen (50 Nachr.)',
|
||||
t['open_chat']!,
|
||||
);
|
||||
report.addTimings(
|
||||
'$mode.search_miss_ms',
|
||||
'[$mode] Suche ohne Treffer (Worst Case)',
|
||||
t['search_miss']!,
|
||||
);
|
||||
report.addTimings(
|
||||
'$mode.search_hit_ms',
|
||||
'[$mode] Suche mit Treffern',
|
||||
t['search_hit']!,
|
||||
);
|
||||
report.addValue(
|
||||
'$mode.size_mb',
|
||||
'[$mode] DB-Größe inkl. WAL',
|
||||
dbSizeMb(paths[mode]!),
|
||||
'MB',
|
||||
);
|
||||
}
|
||||
|
||||
for (final (id, label) in [
|
||||
('write_total_ms', 'Sync-Schreiben'),
|
||||
('startup_ms', 'App-Start'),
|
||||
('open_chat_ms', 'Chat öffnen'),
|
||||
('search_miss_ms', 'Suche ohne Treffer'),
|
||||
]) {
|
||||
final plain = report.valueOf('klartext.$id');
|
||||
final enc = report.valueOf('sqlcipher.$id');
|
||||
report.addValue(
|
||||
'cipher_overhead.$id',
|
||||
'Aufschlag Verschlüsselung: $label',
|
||||
(enc / plain - 1) * 100,
|
||||
'%',
|
||||
);
|
||||
}
|
||||
|
||||
final regressions = await report.finish();
|
||||
if (strictBench) expect(regressions, isEmpty);
|
||||
} finally {
|
||||
try {
|
||||
await tempDir.delete(recursive: true);
|
||||
} catch (_) {}
|
||||
}
|
||||
},
|
||||
skip: skip,
|
||||
timeout: const Timeout(Duration(minutes: 20)),
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
// Benchmarks sind Test-Code, liegen aber bewusst außerhalb von test/ (sie
|
||||
// sollen in einem normalen `flutter test` NICHT mitlaufen) – deshalb kennt
|
||||
// der Analyzer sie nicht als Tests.
|
||||
// ignore_for_file: invalid_use_of_visible_for_testing_member
|
||||
@TestOn('windows')
|
||||
library;
|
||||
|
||||
import 'dart:io';
|
||||
|
||||
import 'package:flutter_test/flutter_test.dart';
|
||||
import 'package:path/path.dart' as p;
|
||||
import 'package:pyramid/core/app_database.dart';
|
||||
|
||||
import '../test/support/synthetic_matrix_data.dart';
|
||||
import 'support/bench.dart';
|
||||
|
||||
/// Migrations-Benchmark: Wie lange dauert die einmalige Umstellung
|
||||
/// Klartext → SQLCipher beim ersten Start nach dem Update (Utas Fall)?
|
||||
///
|
||||
/// Baut eine Klartext-DB über die echte SDK-Speicherschicht, kopiert sie vor
|
||||
/// jedem Durchlauf frisch und misst `AppDatabase.migratePlaintextToEncrypted`
|
||||
/// (exakt der App-Codepfad: Checkpoint, Backup, Export, Verifikation, Tausch).
|
||||
/// Danach wird geprüft, dass die Raumliste aus der verschlüsselten DB wieder
|
||||
/// vollständig lädt. Ergebnis zusätzlich als ms pro MB – damit lässt sich die
|
||||
/// Dauer für eine echte DB-Größe hochrechnen.
|
||||
|
||||
const _benchKey =
|
||||
'5e4d3c2b1a0f9e8d7c6b5a4f3e2d1c0b5e4d3c2b1a0f9e8d7c6b5a4f3e2d1c0b';
|
||||
|
||||
void main() {
|
||||
final skip = findCipherDll() == null
|
||||
? 'SQLCipher-DLL fehlt – erst "flutter build windows" ausführen'
|
||||
: null;
|
||||
|
||||
test(
|
||||
'Migration Klartext → SQLCipher',
|
||||
() async {
|
||||
final rooms = scaled(20);
|
||||
final eventsPerRoom = scaled(1000);
|
||||
final report = BenchReport(
|
||||
'migration',
|
||||
params: {'rooms': rooms, 'eventsPerRoom': eventsPerRoom},
|
||||
);
|
||||
final factory = cipherFactory();
|
||||
final tempDir = await Directory.systemTemp.createTemp(
|
||||
'pyramid_bench_migration',
|
||||
);
|
||||
|
||||
try {
|
||||
// Vorlage: Klartext-DB wie vor der SQLCipher-Einführung.
|
||||
final templatePath = p.join(tempDir.path, 'template.sqlite');
|
||||
var db = await factory.openDatabase(templatePath);
|
||||
await applyAppPragmas(db);
|
||||
var client = await SyntheticMatrixData.openClient(db);
|
||||
await SyntheticMatrixData.populate(
|
||||
client,
|
||||
rooms: rooms,
|
||||
eventsPerRoom: eventsPerRoom,
|
||||
);
|
||||
await client.dispose(closeDatabase: true);
|
||||
// WAL in die Hauptdatei schreiben, damit die Kopie vollständig ist.
|
||||
db = await factory.openDatabase(templatePath);
|
||||
await db.rawQuery('PRAGMA wal_checkpoint(TRUNCATE)');
|
||||
await db.close();
|
||||
final sizeMb = dbSizeMb(templatePath);
|
||||
|
||||
final workPath = p.join(tempDir.path, 'pyramid.sqlite');
|
||||
Future<void> freshCopy() async {
|
||||
for (final suffix in ['', '-wal', '-shm', '.enc', '.premigration']) {
|
||||
final f = File('$workPath$suffix');
|
||||
if (f.existsSync()) f.deleteSync();
|
||||
}
|
||||
await File(templatePath).copy(workPath);
|
||||
}
|
||||
|
||||
final times = await sample(
|
||||
() async {
|
||||
final ok = await AppDatabase.migratePlaintextToEncrypted(
|
||||
factory: factory,
|
||||
dbPath: workPath,
|
||||
hexKey: _benchKey,
|
||||
);
|
||||
if (!ok) throw StateError('Migration fehlgeschlagen');
|
||||
},
|
||||
setup: freshCopy,
|
||||
warmup: 0,
|
||||
runs: 3,
|
||||
);
|
||||
|
||||
// Ergebnis der letzten Migration prüfen: verschlüsselt + vollständig.
|
||||
expect(await AppDatabase.isPlaintextSqlite(File(workPath)), isFalse);
|
||||
db = await AppDatabase.openEncrypted(factory, workPath, _benchKey);
|
||||
client = await SyntheticMatrixData.openClient(db);
|
||||
final loaded = await SyntheticMatrixData.loadRooms(client);
|
||||
final firstPage = await client.database.getEventList(
|
||||
loaded.first,
|
||||
limit: 50,
|
||||
);
|
||||
await client.dispose(closeDatabase: true);
|
||||
expect(loaded, hasLength(rooms));
|
||||
expect(firstPage, hasLength(50));
|
||||
|
||||
report.addValue('db_size_mb', 'DB-Größe vor Migration', sizeMb, 'MB');
|
||||
report.addTimings('migrate_ms', 'Migration gesamt', times);
|
||||
report.addValue(
|
||||
'migrate_ms_per_mb',
|
||||
'Migration je MB',
|
||||
median(times) / sizeMb,
|
||||
'ms/MB',
|
||||
compare: true,
|
||||
);
|
||||
|
||||
final regressions = await report.finish();
|
||||
if (strictBench) expect(regressions, isEmpty);
|
||||
} finally {
|
||||
try {
|
||||
await tempDir.delete(recursive: true);
|
||||
} catch (_) {}
|
||||
}
|
||||
},
|
||||
skip: skip,
|
||||
timeout: const Timeout(Duration(minutes: 20)),
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,271 @@
|
||||
import 'dart:convert';
|
||||
import 'dart:ffi';
|
||||
import 'dart:io';
|
||||
|
||||
import 'package:sqflite_common_ffi/sqflite_ffi.dart';
|
||||
import 'package:sqlite3/open.dart';
|
||||
|
||||
/// Kleines Benchmark-Gerüst für `benchmark/` (bewusst ohne Zusatzpaket).
|
||||
///
|
||||
/// Jede Suite sammelt Messwerte in einem [BenchReport]. Am Ende:
|
||||
/// * Tabelle in der Konsole (inkl. Vergleich mit der Baseline),
|
||||
/// * JSON nach `build/benchmarks/<suite>.json` (für spätere Vergleiche),
|
||||
/// * Vergleich mit `benchmark/baseline/<suite>.json`, falls vorhanden.
|
||||
///
|
||||
/// Umgebungsvariablen:
|
||||
/// * `PYRAMID_BENCH_UPDATE_BASELINE=1` – aktuelle Werte als neue Baseline
|
||||
/// speichern (nach einer bewussten Änderung / auf neuer Hardware).
|
||||
/// * `PYRAMID_BENCH_STRICT=1` – Regressionen lassen den Lauf scheitern
|
||||
/// (sonst nur Warnung, weil Messrauschen normal ist).
|
||||
/// * `PYRAMID_BENCH_SCALE` – Datenmenge skalieren (Standard 1.0;
|
||||
/// z. B. 0.2 für einen schnellen Probelauf, 3 für „Power-User").
|
||||
|
||||
/// Ab so viel Prozent langsamer als die Baseline gilt ein Wert als Regression.
|
||||
const regressionThresholdPercent = 25.0;
|
||||
|
||||
/// Unterhalb dieser absoluten Differenz (ms) wird nie gewarnt – bei sehr
|
||||
/// kleinen Werten ist das reines Rauschen.
|
||||
const regressionFloorMs = 2.0;
|
||||
|
||||
double get benchScale =>
|
||||
double.tryParse(Platform.environment['PYRAMID_BENCH_SCALE'] ?? '') ?? 1.0;
|
||||
|
||||
int scaled(int value) => (value * benchScale).round().clamp(1, 1 << 30);
|
||||
|
||||
/// Führt [body] erst [warmup]-mal ungemessen, dann [runs]-mal gemessen aus
|
||||
/// und liefert die Laufzeiten in Millisekunden. [setup] läuft vor JEDEM
|
||||
/// Durchlauf und wird nicht mitgemessen.
|
||||
Future<List<double>> sample(
|
||||
Future<void> Function() body, {
|
||||
int warmup = 1,
|
||||
int runs = 5,
|
||||
Future<void> Function()? setup,
|
||||
}) async {
|
||||
for (var i = 0; i < warmup; i++) {
|
||||
if (setup != null) await setup();
|
||||
await body();
|
||||
}
|
||||
final out = <double>[];
|
||||
for (var i = 0; i < runs; i++) {
|
||||
if (setup != null) await setup();
|
||||
final sw = Stopwatch()..start();
|
||||
await body();
|
||||
sw.stop();
|
||||
out.add(sw.elapsedMicroseconds / 1000.0);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
double median(List<double> values) {
|
||||
final sorted = [...values]..sort();
|
||||
final mid = sorted.length ~/ 2;
|
||||
return sorted.length.isOdd
|
||||
? sorted[mid]
|
||||
: (sorted[mid - 1] + sorted[mid]) / 2;
|
||||
}
|
||||
|
||||
class BenchMetric {
|
||||
BenchMetric({
|
||||
required this.id,
|
||||
required this.label,
|
||||
required this.unit,
|
||||
required this.value,
|
||||
this.min,
|
||||
this.runs = 1,
|
||||
this.compare = true,
|
||||
});
|
||||
|
||||
final String id;
|
||||
final String label;
|
||||
final String unit;
|
||||
|
||||
/// Median (bzw. Einzelwert) – der Wert, der verglichen wird.
|
||||
final double value;
|
||||
final double? min;
|
||||
final int runs;
|
||||
|
||||
/// `false` für reine Info-Werte (z. B. Dateigröße), die nicht als
|
||||
/// Regression gewertet werden.
|
||||
final bool compare;
|
||||
|
||||
Map<String, Object?> toJson() => {
|
||||
'label': label,
|
||||
'unit': unit,
|
||||
'value': double.parse(value.toStringAsFixed(3)),
|
||||
if (min != null) 'min': double.parse(min!.toStringAsFixed(3)),
|
||||
'runs': runs,
|
||||
'compare': compare,
|
||||
};
|
||||
}
|
||||
|
||||
class BenchReport {
|
||||
BenchReport(this.suite, {this.params = const {}});
|
||||
|
||||
final String suite;
|
||||
final Map<String, Object?> params;
|
||||
final _metrics = <BenchMetric>[];
|
||||
|
||||
/// Laufzeit-Messreihe (ms) als Metrik ablegen.
|
||||
void addTimings(String id, String label, List<double> timingsMs) {
|
||||
_metrics.add(
|
||||
BenchMetric(
|
||||
id: id,
|
||||
label: label,
|
||||
unit: 'ms',
|
||||
value: median(timingsMs),
|
||||
min: timingsMs.reduce((a, b) => a < b ? a : b),
|
||||
runs: timingsMs.length,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
void addValue(
|
||||
String id,
|
||||
String label,
|
||||
double value,
|
||||
String unit, {
|
||||
bool compare = false,
|
||||
}) {
|
||||
_metrics.add(
|
||||
BenchMetric(
|
||||
id: id,
|
||||
label: label,
|
||||
unit: unit,
|
||||
value: value,
|
||||
compare: compare,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
double valueOf(String id) => _metrics.firstWhere((m) => m.id == id).value;
|
||||
|
||||
/// Schreibt Bericht + ggf. Baseline und liefert die Liste der Regressionen
|
||||
/// (leer = alles im Rahmen).
|
||||
Future<List<String>> finish() async {
|
||||
final baselineFile = File('benchmark/baseline/$suite.json');
|
||||
Map<String, Object?>? baseline;
|
||||
if (baselineFile.existsSync()) {
|
||||
final json =
|
||||
jsonDecode(baselineFile.readAsStringSync()) as Map<String, Object?>;
|
||||
baseline = (json['metrics'] as Map).cast<String, Object?>();
|
||||
}
|
||||
|
||||
final regressions = <String>[];
|
||||
final buf = StringBuffer()
|
||||
..writeln()
|
||||
..writeln(
|
||||
'══ Benchmark "$suite" '
|
||||
'${params.isEmpty ? '' : params.entries.map((e) => '${e.key}=${e.value}').join(', ')}',
|
||||
)
|
||||
..writeln(
|
||||
'${'Messung'.padRight(52)}${'Wert'.padLeft(12)} ${'Baseline'.padLeft(10)} Δ',
|
||||
);
|
||||
for (final m in _metrics) {
|
||||
final base = (baseline?[m.id] as Map?)?['value'] as num?;
|
||||
var delta = '';
|
||||
// Für Prozent-Werte (z. B. Verschlüsselungs-Aufschlag) wäre ein
|
||||
// relatives Δ irreführend (4 % → 11 % = „+150 %“) – dort nur die Werte.
|
||||
if (base != null && base > 0 && m.unit != '%') {
|
||||
final pct = (m.value - base) / base * 100;
|
||||
delta = '${pct >= 0 ? '+' : ''}${pct.toStringAsFixed(0)} %';
|
||||
if (m.compare &&
|
||||
pct > regressionThresholdPercent &&
|
||||
(m.unit != 'ms' || m.value - base > regressionFloorMs)) {
|
||||
delta += ' ⚠ LANGSAMER';
|
||||
regressions.add(
|
||||
'${m.label}: ${_fmt(m.value)} ${m.unit} '
|
||||
'(Baseline ${_fmt(base.toDouble())} ${m.unit}, $delta)',
|
||||
);
|
||||
}
|
||||
}
|
||||
buf.writeln(
|
||||
'${m.label.padRight(52)}'
|
||||
'${'${_fmt(m.value)} ${m.unit}'.padLeft(12)} '
|
||||
'${base == null ? '–'.padLeft(10) : _fmt(base.toDouble()).padLeft(10)} $delta',
|
||||
);
|
||||
}
|
||||
// ignore: avoid_print
|
||||
print(buf);
|
||||
|
||||
final report = {
|
||||
'suite': suite,
|
||||
'timestamp': DateTime.now().toIso8601String(),
|
||||
'git': await _gitRevision(),
|
||||
'host': {
|
||||
'os': Platform.operatingSystemVersion,
|
||||
'cpus': Platform.numberOfProcessors,
|
||||
},
|
||||
'params': params,
|
||||
'metrics': {for (final m in _metrics) m.id: m.toJson()},
|
||||
};
|
||||
const encoder = JsonEncoder.withIndent(' ');
|
||||
final outFile = File('build/benchmarks/$suite.json');
|
||||
outFile.parent.createSync(recursive: true);
|
||||
outFile.writeAsStringSync(encoder.convert(report));
|
||||
|
||||
if (Platform.environment['PYRAMID_BENCH_UPDATE_BASELINE'] == '1') {
|
||||
baselineFile.parent.createSync(recursive: true);
|
||||
baselineFile.writeAsStringSync('${encoder.convert(report)}\n');
|
||||
// ignore: avoid_print
|
||||
print('Baseline aktualisiert: ${baselineFile.path}');
|
||||
}
|
||||
return regressions;
|
||||
}
|
||||
|
||||
static String _fmt(double v) =>
|
||||
v >= 100 ? v.toStringAsFixed(0) : v.toStringAsFixed(v >= 10 ? 1 : 2);
|
||||
|
||||
static Future<String?> _gitRevision() async {
|
||||
try {
|
||||
final r = await Process.run('git', ['rev-parse', '--short', 'HEAD']);
|
||||
return r.exitCode == 0 ? (r.stdout as String).trim() : null;
|
||||
} catch (_) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool get strictBench => Platform.environment['PYRAMID_BENCH_STRICT'] == '1';
|
||||
|
||||
// ── SQLCipher aus dem App-Build ────────────────────────────────────────────
|
||||
|
||||
/// Die vom sqlcipher_flutter_libs-Plugin gebaute `sqlite3.dll` – dieselbe
|
||||
/// Bibliothek, die die App benutzt. `null`, wenn noch nie gebaut wurde.
|
||||
String? findCipherDll() {
|
||||
for (final config in ['Release', 'Profile', 'Debug']) {
|
||||
final file = File('build/windows/x64/runner/$config/sqlite3.dll');
|
||||
if (file.existsSync()) return file.absolute.path;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/// Läuft (wie in der App) im Hintergrund-Isolate von sqflite_common_ffi –
|
||||
/// deshalb top-level und ohne eingefangene Variablen.
|
||||
void cipherFfiInit() {
|
||||
final dll = findCipherDll();
|
||||
if (dll == null) throw StateError('SQLCipher-DLL fehlt');
|
||||
open.overrideFor(OperatingSystem.windows, () => DynamicLibrary.open(dll));
|
||||
}
|
||||
|
||||
/// Factory wie in der App: eigenes DB-Isolate, SQLCipher-Bibliothek.
|
||||
/// Ohne `PRAGMA key` verhält sich SQLCipher wie normales SQLite – so laufen
|
||||
/// Klartext- und verschlüsselte Messung mit DERSELBEN Bibliothek und sind
|
||||
/// fair vergleichbar.
|
||||
DatabaseFactory cipherFactory() =>
|
||||
createDatabaseFactoryFfi(ffiInit: cipherFfiInit);
|
||||
|
||||
/// Dieselben PRAGMAs wie `AppDatabase._applyPragmas` (dort privat).
|
||||
Future<void> applyAppPragmas(Database db) async {
|
||||
await db.execute('PRAGMA journal_mode=WAL');
|
||||
await db.execute('PRAGMA busy_timeout = 5000');
|
||||
}
|
||||
|
||||
/// Gesamtgröße der DB inkl. WAL-Sidecar in MB.
|
||||
double dbSizeMb(String path) {
|
||||
var bytes = 0;
|
||||
for (final suffix in ['', '-wal']) {
|
||||
final f = File('$path$suffix');
|
||||
if (f.existsSync()) bytes += f.lengthSync();
|
||||
}
|
||||
return bytes / (1024 * 1024);
|
||||
}
|
||||
Reference in New Issue
Block a user