feat: Benchmarks (DB Klartext vs. SQLCipher, Migration) + scripts/test.ps1
benchmark/ misst Sync-Schreiben, App-Start, Chat oeffnen, lokale Suche und die Migrationsdauer (A-B-B-A, Aufwaermlauf, Median), schreibt build/benchmarks/*.json und vergleicht mit benchmark/baseline/. scripts/test.ps1 fuehrt Analyse, Unit-Tests, optional Live-Test (-Live) und Benchmarks (-Bench) mit einem Befehl aus. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Bernd Steckmeister
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
f4d334a60b
commit
df1b392ed8
@@ -0,0 +1,271 @@
|
||||
import 'dart:convert';
|
||||
import 'dart:ffi';
|
||||
import 'dart:io';
|
||||
|
||||
import 'package:sqflite_common_ffi/sqflite_ffi.dart';
|
||||
import 'package:sqlite3/open.dart';
|
||||
|
||||
/// Kleines Benchmark-Gerüst für `benchmark/` (bewusst ohne Zusatzpaket).
|
||||
///
|
||||
/// Jede Suite sammelt Messwerte in einem [BenchReport]. Am Ende:
|
||||
/// * Tabelle in der Konsole (inkl. Vergleich mit der Baseline),
|
||||
/// * JSON nach `build/benchmarks/<suite>.json` (für spätere Vergleiche),
|
||||
/// * Vergleich mit `benchmark/baseline/<suite>.json`, falls vorhanden.
|
||||
///
|
||||
/// Umgebungsvariablen:
|
||||
/// * `PYRAMID_BENCH_UPDATE_BASELINE=1` – aktuelle Werte als neue Baseline
|
||||
/// speichern (nach einer bewussten Änderung / auf neuer Hardware).
|
||||
/// * `PYRAMID_BENCH_STRICT=1` – Regressionen lassen den Lauf scheitern
|
||||
/// (sonst nur Warnung, weil Messrauschen normal ist).
|
||||
/// * `PYRAMID_BENCH_SCALE` – Datenmenge skalieren (Standard 1.0;
|
||||
/// z. B. 0.2 für einen schnellen Probelauf, 3 für „Power-User").
|
||||
|
||||
/// Ab so viel Prozent langsamer als die Baseline gilt ein Wert als Regression.
|
||||
const regressionThresholdPercent = 25.0;
|
||||
|
||||
/// Unterhalb dieser absoluten Differenz (ms) wird nie gewarnt – bei sehr
|
||||
/// kleinen Werten ist das reines Rauschen.
|
||||
const regressionFloorMs = 2.0;
|
||||
|
||||
double get benchScale =>
|
||||
double.tryParse(Platform.environment['PYRAMID_BENCH_SCALE'] ?? '') ?? 1.0;
|
||||
|
||||
int scaled(int value) => (value * benchScale).round().clamp(1, 1 << 30);
|
||||
|
||||
/// Führt [body] erst [warmup]-mal ungemessen, dann [runs]-mal gemessen aus
|
||||
/// und liefert die Laufzeiten in Millisekunden. [setup] läuft vor JEDEM
|
||||
/// Durchlauf und wird nicht mitgemessen.
|
||||
Future<List<double>> sample(
|
||||
Future<void> Function() body, {
|
||||
int warmup = 1,
|
||||
int runs = 5,
|
||||
Future<void> Function()? setup,
|
||||
}) async {
|
||||
for (var i = 0; i < warmup; i++) {
|
||||
if (setup != null) await setup();
|
||||
await body();
|
||||
}
|
||||
final out = <double>[];
|
||||
for (var i = 0; i < runs; i++) {
|
||||
if (setup != null) await setup();
|
||||
final sw = Stopwatch()..start();
|
||||
await body();
|
||||
sw.stop();
|
||||
out.add(sw.elapsedMicroseconds / 1000.0);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
double median(List<double> values) {
|
||||
final sorted = [...values]..sort();
|
||||
final mid = sorted.length ~/ 2;
|
||||
return sorted.length.isOdd
|
||||
? sorted[mid]
|
||||
: (sorted[mid - 1] + sorted[mid]) / 2;
|
||||
}
|
||||
|
||||
class BenchMetric {
|
||||
BenchMetric({
|
||||
required this.id,
|
||||
required this.label,
|
||||
required this.unit,
|
||||
required this.value,
|
||||
this.min,
|
||||
this.runs = 1,
|
||||
this.compare = true,
|
||||
});
|
||||
|
||||
final String id;
|
||||
final String label;
|
||||
final String unit;
|
||||
|
||||
/// Median (bzw. Einzelwert) – der Wert, der verglichen wird.
|
||||
final double value;
|
||||
final double? min;
|
||||
final int runs;
|
||||
|
||||
/// `false` für reine Info-Werte (z. B. Dateigröße), die nicht als
|
||||
/// Regression gewertet werden.
|
||||
final bool compare;
|
||||
|
||||
Map<String, Object?> toJson() => {
|
||||
'label': label,
|
||||
'unit': unit,
|
||||
'value': double.parse(value.toStringAsFixed(3)),
|
||||
if (min != null) 'min': double.parse(min!.toStringAsFixed(3)),
|
||||
'runs': runs,
|
||||
'compare': compare,
|
||||
};
|
||||
}
|
||||
|
||||
class BenchReport {
|
||||
BenchReport(this.suite, {this.params = const {}});
|
||||
|
||||
final String suite;
|
||||
final Map<String, Object?> params;
|
||||
final _metrics = <BenchMetric>[];
|
||||
|
||||
/// Laufzeit-Messreihe (ms) als Metrik ablegen.
|
||||
void addTimings(String id, String label, List<double> timingsMs) {
|
||||
_metrics.add(
|
||||
BenchMetric(
|
||||
id: id,
|
||||
label: label,
|
||||
unit: 'ms',
|
||||
value: median(timingsMs),
|
||||
min: timingsMs.reduce((a, b) => a < b ? a : b),
|
||||
runs: timingsMs.length,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
void addValue(
|
||||
String id,
|
||||
String label,
|
||||
double value,
|
||||
String unit, {
|
||||
bool compare = false,
|
||||
}) {
|
||||
_metrics.add(
|
||||
BenchMetric(
|
||||
id: id,
|
||||
label: label,
|
||||
unit: unit,
|
||||
value: value,
|
||||
compare: compare,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
double valueOf(String id) => _metrics.firstWhere((m) => m.id == id).value;
|
||||
|
||||
/// Schreibt Bericht + ggf. Baseline und liefert die Liste der Regressionen
|
||||
/// (leer = alles im Rahmen).
|
||||
Future<List<String>> finish() async {
|
||||
final baselineFile = File('benchmark/baseline/$suite.json');
|
||||
Map<String, Object?>? baseline;
|
||||
if (baselineFile.existsSync()) {
|
||||
final json =
|
||||
jsonDecode(baselineFile.readAsStringSync()) as Map<String, Object?>;
|
||||
baseline = (json['metrics'] as Map).cast<String, Object?>();
|
||||
}
|
||||
|
||||
final regressions = <String>[];
|
||||
final buf = StringBuffer()
|
||||
..writeln()
|
||||
..writeln(
|
||||
'══ Benchmark "$suite" '
|
||||
'${params.isEmpty ? '' : params.entries.map((e) => '${e.key}=${e.value}').join(', ')}',
|
||||
)
|
||||
..writeln(
|
||||
'${'Messung'.padRight(52)}${'Wert'.padLeft(12)} ${'Baseline'.padLeft(10)} Δ',
|
||||
);
|
||||
for (final m in _metrics) {
|
||||
final base = (baseline?[m.id] as Map?)?['value'] as num?;
|
||||
var delta = '';
|
||||
// Für Prozent-Werte (z. B. Verschlüsselungs-Aufschlag) wäre ein
|
||||
// relatives Δ irreführend (4 % → 11 % = „+150 %“) – dort nur die Werte.
|
||||
if (base != null && base > 0 && m.unit != '%') {
|
||||
final pct = (m.value - base) / base * 100;
|
||||
delta = '${pct >= 0 ? '+' : ''}${pct.toStringAsFixed(0)} %';
|
||||
if (m.compare &&
|
||||
pct > regressionThresholdPercent &&
|
||||
(m.unit != 'ms' || m.value - base > regressionFloorMs)) {
|
||||
delta += ' ⚠ LANGSAMER';
|
||||
regressions.add(
|
||||
'${m.label}: ${_fmt(m.value)} ${m.unit} '
|
||||
'(Baseline ${_fmt(base.toDouble())} ${m.unit}, $delta)',
|
||||
);
|
||||
}
|
||||
}
|
||||
buf.writeln(
|
||||
'${m.label.padRight(52)}'
|
||||
'${'${_fmt(m.value)} ${m.unit}'.padLeft(12)} '
|
||||
'${base == null ? '–'.padLeft(10) : _fmt(base.toDouble()).padLeft(10)} $delta',
|
||||
);
|
||||
}
|
||||
// ignore: avoid_print
|
||||
print(buf);
|
||||
|
||||
final report = {
|
||||
'suite': suite,
|
||||
'timestamp': DateTime.now().toIso8601String(),
|
||||
'git': await _gitRevision(),
|
||||
'host': {
|
||||
'os': Platform.operatingSystemVersion,
|
||||
'cpus': Platform.numberOfProcessors,
|
||||
},
|
||||
'params': params,
|
||||
'metrics': {for (final m in _metrics) m.id: m.toJson()},
|
||||
};
|
||||
const encoder = JsonEncoder.withIndent(' ');
|
||||
final outFile = File('build/benchmarks/$suite.json');
|
||||
outFile.parent.createSync(recursive: true);
|
||||
outFile.writeAsStringSync(encoder.convert(report));
|
||||
|
||||
if (Platform.environment['PYRAMID_BENCH_UPDATE_BASELINE'] == '1') {
|
||||
baselineFile.parent.createSync(recursive: true);
|
||||
baselineFile.writeAsStringSync('${encoder.convert(report)}\n');
|
||||
// ignore: avoid_print
|
||||
print('Baseline aktualisiert: ${baselineFile.path}');
|
||||
}
|
||||
return regressions;
|
||||
}
|
||||
|
||||
static String _fmt(double v) =>
|
||||
v >= 100 ? v.toStringAsFixed(0) : v.toStringAsFixed(v >= 10 ? 1 : 2);
|
||||
|
||||
static Future<String?> _gitRevision() async {
|
||||
try {
|
||||
final r = await Process.run('git', ['rev-parse', '--short', 'HEAD']);
|
||||
return r.exitCode == 0 ? (r.stdout as String).trim() : null;
|
||||
} catch (_) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool get strictBench => Platform.environment['PYRAMID_BENCH_STRICT'] == '1';
|
||||
|
||||
// ── SQLCipher aus dem App-Build ────────────────────────────────────────────
|
||||
|
||||
/// Die vom sqlcipher_flutter_libs-Plugin gebaute `sqlite3.dll` – dieselbe
|
||||
/// Bibliothek, die die App benutzt. `null`, wenn noch nie gebaut wurde.
|
||||
String? findCipherDll() {
|
||||
for (final config in ['Release', 'Profile', 'Debug']) {
|
||||
final file = File('build/windows/x64/runner/$config/sqlite3.dll');
|
||||
if (file.existsSync()) return file.absolute.path;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/// Läuft (wie in der App) im Hintergrund-Isolate von sqflite_common_ffi –
|
||||
/// deshalb top-level und ohne eingefangene Variablen.
|
||||
void cipherFfiInit() {
|
||||
final dll = findCipherDll();
|
||||
if (dll == null) throw StateError('SQLCipher-DLL fehlt');
|
||||
open.overrideFor(OperatingSystem.windows, () => DynamicLibrary.open(dll));
|
||||
}
|
||||
|
||||
/// Factory wie in der App: eigenes DB-Isolate, SQLCipher-Bibliothek.
|
||||
/// Ohne `PRAGMA key` verhält sich SQLCipher wie normales SQLite – so laufen
|
||||
/// Klartext- und verschlüsselte Messung mit DERSELBEN Bibliothek und sind
|
||||
/// fair vergleichbar.
|
||||
DatabaseFactory cipherFactory() =>
|
||||
createDatabaseFactoryFfi(ffiInit: cipherFfiInit);
|
||||
|
||||
/// Dieselben PRAGMAs wie `AppDatabase._applyPragmas` (dort privat).
|
||||
Future<void> applyAppPragmas(Database db) async {
|
||||
await db.execute('PRAGMA journal_mode=WAL');
|
||||
await db.execute('PRAGMA busy_timeout = 5000');
|
||||
}
|
||||
|
||||
/// Gesamtgröße der DB inkl. WAL-Sidecar in MB.
|
||||
double dbSizeMb(String path) {
|
||||
var bytes = 0;
|
||||
for (final suffix in ['', '-wal']) {
|
||||
final f = File('$path$suffix');
|
||||
if (f.existsSync()) bytes += f.lengthSync();
|
||||
}
|
||||
return bytes / (1024 * 1024);
|
||||
}
|
||||
Reference in New Issue
Block a user