Bernd SteckmeisterandClaude Opus 5.5
df1b392ed8
benchmark/ misst Sync-Schreiben, App-Start, Chat oeffnen, lokale Suche und die Migrationsdauer (A-B-B-A, Aufwaermlauf, Median), schreibt build/benchmarks/*.json und vergleicht mit benchmark/baseline/. scripts/test.ps1 fuehrt Analyse, Unit-Tests, optional Live-Test (-Live) und Benchmarks (-Bench) mit einem Befehl aus. Co-Authored-By: Claude Opus 5.5 <[email protected]>
272 lines
8.6 KiB
Dart
272 lines
8.6 KiB
Dart
import 'dart:convert';
|
||
import 'dart:ffi';
|
||
import 'dart:io';
|
||
|
||
import 'package:sqflite_common_ffi/sqflite_ffi.dart';
|
||
import 'package:sqlite3/open.dart';
|
||
|
||
/// Kleines Benchmark-Gerüst für `benchmark/` (bewusst ohne Zusatzpaket).
|
||
///
|
||
/// Jede Suite sammelt Messwerte in einem [BenchReport]. Am Ende:
|
||
/// * Tabelle in der Konsole (inkl. Vergleich mit der Baseline),
|
||
/// * JSON nach `build/benchmarks/<suite>.json` (für spätere Vergleiche),
|
||
/// * Vergleich mit `benchmark/baseline/<suite>.json`, falls vorhanden.
|
||
///
|
||
/// Umgebungsvariablen:
|
||
/// * `PYRAMID_BENCH_UPDATE_BASELINE=1` – aktuelle Werte als neue Baseline
|
||
/// speichern (nach einer bewussten Änderung / auf neuer Hardware).
|
||
/// * `PYRAMID_BENCH_STRICT=1` – Regressionen lassen den Lauf scheitern
|
||
/// (sonst nur Warnung, weil Messrauschen normal ist).
|
||
/// * `PYRAMID_BENCH_SCALE` – Datenmenge skalieren (Standard 1.0;
|
||
/// z. B. 0.2 für einen schnellen Probelauf, 3 für „Power-User").
|
||
|
||
/// Ab so viel Prozent langsamer als die Baseline gilt ein Wert als Regression.
|
||
const regressionThresholdPercent = 25.0;
|
||
|
||
/// Unterhalb dieser absoluten Differenz (ms) wird nie gewarnt – bei sehr
|
||
/// kleinen Werten ist das reines Rauschen.
|
||
const regressionFloorMs = 2.0;
|
||
|
||
double get benchScale =>
|
||
double.tryParse(Platform.environment['PYRAMID_BENCH_SCALE'] ?? '') ?? 1.0;
|
||
|
||
int scaled(int value) => (value * benchScale).round().clamp(1, 1 << 30);
|
||
|
||
/// Führt [body] erst [warmup]-mal ungemessen, dann [runs]-mal gemessen aus
|
||
/// und liefert die Laufzeiten in Millisekunden. [setup] läuft vor JEDEM
|
||
/// Durchlauf und wird nicht mitgemessen.
|
||
Future<List<double>> sample(
|
||
Future<void> Function() body, {
|
||
int warmup = 1,
|
||
int runs = 5,
|
||
Future<void> Function()? setup,
|
||
}) async {
|
||
for (var i = 0; i < warmup; i++) {
|
||
if (setup != null) await setup();
|
||
await body();
|
||
}
|
||
final out = <double>[];
|
||
for (var i = 0; i < runs; i++) {
|
||
if (setup != null) await setup();
|
||
final sw = Stopwatch()..start();
|
||
await body();
|
||
sw.stop();
|
||
out.add(sw.elapsedMicroseconds / 1000.0);
|
||
}
|
||
return out;
|
||
}
|
||
|
||
double median(List<double> values) {
|
||
final sorted = [...values]..sort();
|
||
final mid = sorted.length ~/ 2;
|
||
return sorted.length.isOdd
|
||
? sorted[mid]
|
||
: (sorted[mid - 1] + sorted[mid]) / 2;
|
||
}
|
||
|
||
class BenchMetric {
|
||
BenchMetric({
|
||
required this.id,
|
||
required this.label,
|
||
required this.unit,
|
||
required this.value,
|
||
this.min,
|
||
this.runs = 1,
|
||
this.compare = true,
|
||
});
|
||
|
||
final String id;
|
||
final String label;
|
||
final String unit;
|
||
|
||
/// Median (bzw. Einzelwert) – der Wert, der verglichen wird.
|
||
final double value;
|
||
final double? min;
|
||
final int runs;
|
||
|
||
/// `false` für reine Info-Werte (z. B. Dateigröße), die nicht als
|
||
/// Regression gewertet werden.
|
||
final bool compare;
|
||
|
||
Map<String, Object?> toJson() => {
|
||
'label': label,
|
||
'unit': unit,
|
||
'value': double.parse(value.toStringAsFixed(3)),
|
||
if (min != null) 'min': double.parse(min!.toStringAsFixed(3)),
|
||
'runs': runs,
|
||
'compare': compare,
|
||
};
|
||
}
|
||
|
||
class BenchReport {
|
||
BenchReport(this.suite, {this.params = const {}});
|
||
|
||
final String suite;
|
||
final Map<String, Object?> params;
|
||
final _metrics = <BenchMetric>[];
|
||
|
||
/// Laufzeit-Messreihe (ms) als Metrik ablegen.
|
||
void addTimings(String id, String label, List<double> timingsMs) {
|
||
_metrics.add(
|
||
BenchMetric(
|
||
id: id,
|
||
label: label,
|
||
unit: 'ms',
|
||
value: median(timingsMs),
|
||
min: timingsMs.reduce((a, b) => a < b ? a : b),
|
||
runs: timingsMs.length,
|
||
),
|
||
);
|
||
}
|
||
|
||
void addValue(
|
||
String id,
|
||
String label,
|
||
double value,
|
||
String unit, {
|
||
bool compare = false,
|
||
}) {
|
||
_metrics.add(
|
||
BenchMetric(
|
||
id: id,
|
||
label: label,
|
||
unit: unit,
|
||
value: value,
|
||
compare: compare,
|
||
),
|
||
);
|
||
}
|
||
|
||
double valueOf(String id) => _metrics.firstWhere((m) => m.id == id).value;
|
||
|
||
/// Schreibt Bericht + ggf. Baseline und liefert die Liste der Regressionen
|
||
/// (leer = alles im Rahmen).
|
||
Future<List<String>> finish() async {
|
||
final baselineFile = File('benchmark/baseline/$suite.json');
|
||
Map<String, Object?>? baseline;
|
||
if (baselineFile.existsSync()) {
|
||
final json =
|
||
jsonDecode(baselineFile.readAsStringSync()) as Map<String, Object?>;
|
||
baseline = (json['metrics'] as Map).cast<String, Object?>();
|
||
}
|
||
|
||
final regressions = <String>[];
|
||
final buf = StringBuffer()
|
||
..writeln()
|
||
..writeln(
|
||
'══ Benchmark "$suite" '
|
||
'${params.isEmpty ? '' : params.entries.map((e) => '${e.key}=${e.value}').join(', ')}',
|
||
)
|
||
..writeln(
|
||
'${'Messung'.padRight(52)}${'Wert'.padLeft(12)} ${'Baseline'.padLeft(10)} Δ',
|
||
);
|
||
for (final m in _metrics) {
|
||
final base = (baseline?[m.id] as Map?)?['value'] as num?;
|
||
var delta = '';
|
||
// Für Prozent-Werte (z. B. Verschlüsselungs-Aufschlag) wäre ein
|
||
// relatives Δ irreführend (4 % → 11 % = „+150 %“) – dort nur die Werte.
|
||
if (base != null && base > 0 && m.unit != '%') {
|
||
final pct = (m.value - base) / base * 100;
|
||
delta = '${pct >= 0 ? '+' : ''}${pct.toStringAsFixed(0)} %';
|
||
if (m.compare &&
|
||
pct > regressionThresholdPercent &&
|
||
(m.unit != 'ms' || m.value - base > regressionFloorMs)) {
|
||
delta += ' ⚠ LANGSAMER';
|
||
regressions.add(
|
||
'${m.label}: ${_fmt(m.value)} ${m.unit} '
|
||
'(Baseline ${_fmt(base.toDouble())} ${m.unit}, $delta)',
|
||
);
|
||
}
|
||
}
|
||
buf.writeln(
|
||
'${m.label.padRight(52)}'
|
||
'${'${_fmt(m.value)} ${m.unit}'.padLeft(12)} '
|
||
'${base == null ? '–'.padLeft(10) : _fmt(base.toDouble()).padLeft(10)} $delta',
|
||
);
|
||
}
|
||
// ignore: avoid_print
|
||
print(buf);
|
||
|
||
final report = {
|
||
'suite': suite,
|
||
'timestamp': DateTime.now().toIso8601String(),
|
||
'git': await _gitRevision(),
|
||
'host': {
|
||
'os': Platform.operatingSystemVersion,
|
||
'cpus': Platform.numberOfProcessors,
|
||
},
|
||
'params': params,
|
||
'metrics': {for (final m in _metrics) m.id: m.toJson()},
|
||
};
|
||
const encoder = JsonEncoder.withIndent(' ');
|
||
final outFile = File('build/benchmarks/$suite.json');
|
||
outFile.parent.createSync(recursive: true);
|
||
outFile.writeAsStringSync(encoder.convert(report));
|
||
|
||
if (Platform.environment['PYRAMID_BENCH_UPDATE_BASELINE'] == '1') {
|
||
baselineFile.parent.createSync(recursive: true);
|
||
baselineFile.writeAsStringSync('${encoder.convert(report)}\n');
|
||
// ignore: avoid_print
|
||
print('Baseline aktualisiert: ${baselineFile.path}');
|
||
}
|
||
return regressions;
|
||
}
|
||
|
||
static String _fmt(double v) =>
|
||
v >= 100 ? v.toStringAsFixed(0) : v.toStringAsFixed(v >= 10 ? 1 : 2);
|
||
|
||
static Future<String?> _gitRevision() async {
|
||
try {
|
||
final r = await Process.run('git', ['rev-parse', '--short', 'HEAD']);
|
||
return r.exitCode == 0 ? (r.stdout as String).trim() : null;
|
||
} catch (_) {
|
||
return null;
|
||
}
|
||
}
|
||
}
|
||
|
||
bool get strictBench => Platform.environment['PYRAMID_BENCH_STRICT'] == '1';
|
||
|
||
// ── SQLCipher aus dem App-Build ────────────────────────────────────────────
|
||
|
||
/// Die vom sqlcipher_flutter_libs-Plugin gebaute `sqlite3.dll` – dieselbe
|
||
/// Bibliothek, die die App benutzt. `null`, wenn noch nie gebaut wurde.
|
||
String? findCipherDll() {
|
||
for (final config in ['Release', 'Profile', 'Debug']) {
|
||
final file = File('build/windows/x64/runner/$config/sqlite3.dll');
|
||
if (file.existsSync()) return file.absolute.path;
|
||
}
|
||
return null;
|
||
}
|
||
|
||
/// Läuft (wie in der App) im Hintergrund-Isolate von sqflite_common_ffi –
|
||
/// deshalb top-level und ohne eingefangene Variablen.
|
||
void cipherFfiInit() {
|
||
final dll = findCipherDll();
|
||
if (dll == null) throw StateError('SQLCipher-DLL fehlt');
|
||
open.overrideFor(OperatingSystem.windows, () => DynamicLibrary.open(dll));
|
||
}
|
||
|
||
/// Factory wie in der App: eigenes DB-Isolate, SQLCipher-Bibliothek.
|
||
/// Ohne `PRAGMA key` verhält sich SQLCipher wie normales SQLite – so laufen
|
||
/// Klartext- und verschlüsselte Messung mit DERSELBEN Bibliothek und sind
|
||
/// fair vergleichbar.
|
||
DatabaseFactory cipherFactory() =>
|
||
createDatabaseFactoryFfi(ffiInit: cipherFfiInit);
|
||
|
||
/// Dieselben PRAGMAs wie `AppDatabase._applyPragmas` (dort privat).
|
||
Future<void> applyAppPragmas(Database db) async {
|
||
await db.execute('PRAGMA journal_mode=WAL');
|
||
await db.execute('PRAGMA busy_timeout = 5000');
|
||
}
|
||
|
||
/// Gesamtgröße der DB inkl. WAL-Sidecar in MB.
|
||
double dbSizeMb(String path) {
|
||
var bytes = 0;
|
||
for (final suffix in ['', '-wal']) {
|
||
final f = File('$path$suffix');
|
||
if (f.existsSync()) bytes += f.lengthSync();
|
||
}
|
||
return bytes / (1024 * 1024);
|
||
}
|