wip: Sicherungs-Commit aller Änderungen seit April + Arbeitsstruktur (CLAUDE.md, ROADMAP.md, PROGRESS.md, Autopilot)
6 Wochen uncommittete Arbeit (Voice-Channels, LiveKit-Manager, Settings-Modal u.v.m.) als ein WIP-Commit gesichert, damit nichts verloren geht und der Pi den aktuellen Stand klonen kann. Thematische Aufarbeitung: siehe ROADMAP M0. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CPrAGBxBT6GfPXzeWQ4AXb
This commit is contained in:
Bernd Steckmeister
@@ -0,0 +1,422 @@
|
||||
# 08 — VoIP, Calls & LiveKit (MatrixRTC)
|
||||
|
||||
---
|
||||
|
||||
## Inhaltsverzeichnis
|
||||
- [Architektur-Überblick](#architektur-überblick)
|
||||
- [1:1 Call (Legacy m.call)](#11-call-legacy-mcall)
|
||||
- [MatrixRTC (Gruppen-Calls)](#matrixrtc-gruppen-calls)
|
||||
- [LiveKit-Integration](#livekit-integration)
|
||||
- [Klingel-Flow](#klingel-flow)
|
||||
- [WebRTC Setup in Flutter](#webrtc-setup-in-flutter)
|
||||
- [Call-State Management](#call-state-management)
|
||||
- [Pyramid VoIP-Architektur](#pyramid-voip-architektur)
|
||||
|
||||
---
|
||||
|
||||
## Architektur-Überblick
|
||||
|
||||
```
|
||||
Matrix Signaling (to-device / room events)
|
||||
│
|
||||
▼
|
||||
MatrixRTC Layer
|
||||
┌─────────────────────────────────┐
|
||||
│ com.famedly.call.member Events │
|
||||
│ (Membership, Backend, Expiry) │
|
||||
└──────────────┬──────────────────┘
|
||||
│
|
||||
┌────────┴────────┐
|
||||
│ │
|
||||
Mesh (P2P) LiveKit (SFU)
|
||||
WebRTC direct livekit-server
|
||||
(1:1, max ~4) (skalierbar, n:m)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 1:1 Call (Legacy m.call)
|
||||
|
||||
```dart
|
||||
// ── Anruf starten ────────────────────────────────────────────────────────
|
||||
// Wird vom SDK über room events (m.call.invite, m.call.candidates,
|
||||
// m.call.answer, m.call.hangup) signalisiert.
|
||||
|
||||
// Anruf-Einladung senden (nach WebRTC offer)
|
||||
await room.sendEvent(
|
||||
{
|
||||
'call_id': 'eindeutige-call-id',
|
||||
'version': 1,
|
||||
'lifetime': 60000, // ms, wie lange das Invite gültig ist
|
||||
'offer': {
|
||||
'type': 'offer',
|
||||
'sdp': sdpString, // WebRTC SDP Offer
|
||||
},
|
||||
},
|
||||
EventTypes.CallInvite, // 'm.call.invite'
|
||||
);
|
||||
|
||||
// ICE-Kandidaten senden
|
||||
await room.sendEvent(
|
||||
{
|
||||
'call_id': callId,
|
||||
'version': 1,
|
||||
'candidates': [
|
||||
{
|
||||
'candidate': 'candidate:...',
|
||||
'sdpMid': '0',
|
||||
'sdpMLineIndex': 0,
|
||||
},
|
||||
],
|
||||
},
|
||||
EventTypes.CallCandidates, // 'm.call.candidates'
|
||||
);
|
||||
|
||||
// Anruf annehmen (SDP Answer)
|
||||
await room.sendEvent(
|
||||
{
|
||||
'call_id': callId,
|
||||
'version': 1,
|
||||
'answer': {
|
||||
'type': 'answer',
|
||||
'sdp': sdpAnswerString,
|
||||
},
|
||||
},
|
||||
EventTypes.CallAnswer, // 'm.call.answer'
|
||||
);
|
||||
|
||||
// Auflegen
|
||||
await room.sendEvent(
|
||||
{
|
||||
'call_id': callId,
|
||||
'version': 1,
|
||||
'reason': 'user_hangup', // oder 'ice_failed', 'invite_timeout'
|
||||
},
|
||||
EventTypes.CallHangup, // 'm.call.hangup'
|
||||
);
|
||||
|
||||
// Ablehnen (Reject)
|
||||
await room.sendEvent(
|
||||
{
|
||||
'call_id': callId,
|
||||
'version': 1,
|
||||
'reason': 'user_hangup',
|
||||
},
|
||||
EventTypes.CallReject, // 'm.call.reject'
|
||||
);
|
||||
|
||||
// Eingehende Anrufe überwachen
|
||||
client.onCallInvite.stream.listen((CallInvite invite) {
|
||||
print('Eingehender Anruf von ${invite.roomId}');
|
||||
print('Call-ID: ${invite.callId}');
|
||||
});
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## MatrixRTC (Gruppen-Calls)
|
||||
|
||||
```dart
|
||||
// MatrixRTC verwendet com.famedly.call.member State-Events
|
||||
|
||||
// ── Call beitreten (Membership senden) ───────────────────────────────────
|
||||
|
||||
final myDeviceId = client.deviceID!;
|
||||
final callId = room.id; // Normalerweise = Raum-ID
|
||||
|
||||
// Membership-Event senden (signalisiert "ich bin im Call")
|
||||
await room.client.sendState(
|
||||
room.id,
|
||||
'org.matrix.msc3401.call.member', // oder 'com.famedly.call.member'
|
||||
{
|
||||
'application': 'm.call',
|
||||
'call_id': callId,
|
||||
'device_id': myDeviceId,
|
||||
'expires_ts': DateTime.now()
|
||||
.add(const Duration(hours: 1))
|
||||
.millisecondsSinceEpoch,
|
||||
'backend': {
|
||||
'type': 'm.sfu', // LiveKit
|
||||
'livekit_service_url': 'https://livekit.example.org',
|
||||
},
|
||||
// Für Mesh:
|
||||
// 'backend': {'type': 'm.mesh'},
|
||||
'membershipID': 'eindeutige-membership-id',
|
||||
},
|
||||
stateKey: '$myDeviceId:$callId',
|
||||
);
|
||||
|
||||
// ── Call verlassen (Membership entfernen) ─────────────────────────────────
|
||||
|
||||
await room.client.sendState(
|
||||
room.id,
|
||||
'org.matrix.msc3401.call.member',
|
||||
{}, // Leeres Content = Austritt
|
||||
stateKey: '$myDeviceId:$callId',
|
||||
);
|
||||
|
||||
// ── Aktive Call-Mitglieder auflesen ──────────────────────────────────────
|
||||
|
||||
final callStates = room.states['org.matrix.msc3401.call.member'];
|
||||
final now = DateTime.now().millisecondsSinceEpoch;
|
||||
final activeMembers = callStates?.values.where((event) {
|
||||
final expiresTs = event.content['expires_ts'] as int? ?? 0;
|
||||
return expiresTs > now; // Nicht abgelaufen
|
||||
}).toList() ?? [];
|
||||
|
||||
for (final member in activeMembers) {
|
||||
print(member.content['device_id']);
|
||||
print(member.stateKey); // 'deviceId:callId'
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## LiveKit-Integration
|
||||
|
||||
```dart
|
||||
// LiveKit-Token vom Matrix-Homeserver anfragen (OpenID)
|
||||
// Dieser Flow authentifiziert den User beim LiveKit-Server ohne eigenes Passwort.
|
||||
|
||||
// 1. OpenID-Token vom Homeserver holen
|
||||
final openIdToken = await client.requestOpenIdToken(client.userID!);
|
||||
// openIdToken.accessToken, .matrixServerName
|
||||
|
||||
// 2. LiveKit-Token beim MatrixRTC-Service tauschen
|
||||
// (Pyramid sendet openIdToken an livekit_service_url des Raums)
|
||||
final response = await http.post(
|
||||
Uri.parse('https://livekit-service.example.org/sfu/get'),
|
||||
headers: {'Content-Type': 'application/json'},
|
||||
body: jsonEncode({
|
||||
'room': room.id,
|
||||
'openid_token': {
|
||||
'access_token': openIdToken.accessToken,
|
||||
'token_type': 'Bearer',
|
||||
'matrix_server_name': openIdToken.matrixServerName,
|
||||
},
|
||||
'device_id': client.deviceID,
|
||||
}),
|
||||
);
|
||||
final livekitToken = jsonDecode(response.body)['token'];
|
||||
|
||||
// 3. Mit LiveKit verbinden
|
||||
import 'package:livekit_client/livekit_client.dart';
|
||||
final room = await LiveKitClient.connect(
|
||||
'wss://livekit.example.org',
|
||||
livekitToken,
|
||||
roomOptions: const RoomOptions(
|
||||
adaptiveStream: true,
|
||||
dynacast: true,
|
||||
defaultVideoPublishOptions: VideoPublishOptions(
|
||||
simulcast: true,
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
// 4. Mikrofon aktivieren
|
||||
await room.localParticipant?.setMicrophoneEnabled(true);
|
||||
|
||||
// 5. Kamera aktivieren (Video-Call)
|
||||
await room.localParticipant?.setCameraEnabled(true);
|
||||
|
||||
// 6. Teilnehmer observieren
|
||||
room.addListener(() {
|
||||
for (final participant in room.participants.values) {
|
||||
print(participant.identity); // Matrix User-ID
|
||||
for (final track in participant.videoTracks) {
|
||||
// Widget: VideoTrackRenderer(track.track!)
|
||||
}
|
||||
}
|
||||
});
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Klingel-Flow
|
||||
|
||||
```dart
|
||||
// ── Anruf initiieren ──────────────────────────────────────────────────────
|
||||
|
||||
// 1. Membership-Event senden (MatrixRTC)
|
||||
// 2. To-Device-Event senden für Push-Benachrichtigung beim Empfänger
|
||||
|
||||
// Klingelanruf via to-device-Event
|
||||
await client.sendToDevice(
|
||||
'com.famedly.call', // Event-Typ
|
||||
{
|
||||
otherUserId: {
|
||||
'*': { // Alle Geräte des Users
|
||||
'call_id': callId,
|
||||
'room_id': room.id,
|
||||
'type': 'invite',
|
||||
'sender_session_id': client.deviceID,
|
||||
},
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
// ── Eingehenden Anruf empfangen ───────────────────────────────────────────
|
||||
|
||||
client.onToDeviceEvent.stream
|
||||
.where((e) => e.type == 'com.famedly.call')
|
||||
.listen((event) {
|
||||
final callId = event.content['call_id'];
|
||||
final roomId = event.content['room_id'];
|
||||
final type = event.content['type']; // 'invite' | 'hangup'
|
||||
// Klingelton abspielen, UI anzeigen
|
||||
});
|
||||
|
||||
// ── Anruf ablehnen (während Klingeln) ────────────────────────────────────
|
||||
|
||||
await client.sendToDevice(
|
||||
'com.famedly.call',
|
||||
{
|
||||
callerId: {'*': {'call_id': callId, 'type': 'reject'}},
|
||||
},
|
||||
);
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## WebRTC Setup in Flutter
|
||||
|
||||
```dart
|
||||
// flutter_webrtc Package
|
||||
import 'package:flutter_webrtc/flutter_webrtc.dart';
|
||||
|
||||
// Media-Stream holen
|
||||
final stream = await navigator.mediaDevices.getUserMedia({
|
||||
'audio': true,
|
||||
'video': false, // true für Video-Call
|
||||
});
|
||||
|
||||
// PeerConnection erstellen
|
||||
final pc = await createPeerConnection({
|
||||
'iceServers': [
|
||||
{'urls': 'stun:stun.matrix.org:3478'},
|
||||
// TURN-Server (für NAT-Traversal):
|
||||
{
|
||||
'urls': 'turn:turn.example.org:5349',
|
||||
'username': 'user',
|
||||
'credential': 'pass',
|
||||
},
|
||||
],
|
||||
'iceCandidatePoolSize': 10,
|
||||
});
|
||||
|
||||
// Track hinzufügen
|
||||
for (final track in stream.getTracks()) {
|
||||
await pc.addTrack(track, stream);
|
||||
}
|
||||
|
||||
// ICE-Kandidaten sammeln
|
||||
pc.onIceCandidate = (RTCIceCandidate candidate) {
|
||||
// Kandidaten via Matrix senden
|
||||
sendIceCandidates([candidate]);
|
||||
};
|
||||
|
||||
// Angebot erstellen
|
||||
final offer = await pc.createOffer();
|
||||
await pc.setLocalDescription(offer);
|
||||
// offer.sdp → via Matrix senden
|
||||
|
||||
// Angebot annehmen
|
||||
await pc.setRemoteDescription(RTCSessionDescription(sdpOffer, 'offer'));
|
||||
final answer = await pc.createAnswer();
|
||||
await pc.setLocalDescription(answer);
|
||||
// answer.sdp → via Matrix senden
|
||||
|
||||
// Eingehende Streams
|
||||
pc.onTrack = (RTCTrackEvent event) {
|
||||
final remoteStream = event.streams.first;
|
||||
// Widget: RTCVideoRenderer() mit remoteStream
|
||||
};
|
||||
|
||||
// Aufräumen
|
||||
await pc.close();
|
||||
await stream.dispose();
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Call-State Management
|
||||
|
||||
```dart
|
||||
// Call-Zustand in Pyramid
|
||||
enum CallState {
|
||||
idle,
|
||||
calling, // Anruf geht raus
|
||||
ringing, // Eingehender Anruf
|
||||
connecting, // ICE/SDP in Bearbeitung
|
||||
connected, // Verbunden
|
||||
ended,
|
||||
}
|
||||
|
||||
// Stummschalten
|
||||
final audioTrack = localStream.getAudioTracks().first;
|
||||
audioTrack.enabled = false; // Mute
|
||||
audioTrack.enabled = true; // Unmute
|
||||
|
||||
// Lautsprecher vs. Hörer (Android/iOS)
|
||||
// flutter_webrtc kümmert sich darum via Helper.setSpeakerphoneOn
|
||||
|
||||
// DTMF (Zifferntasten)
|
||||
final sender = pc.getSenders().firstWhere(
|
||||
(s) => s.track?.kind == 'audio');
|
||||
await sender.dtmf?.insertDTMF('1');
|
||||
|
||||
// Screen-Sharing starten
|
||||
final screenStream = await navigator.mediaDevices.getDisplayMedia({
|
||||
'video': true,
|
||||
'audio': false,
|
||||
});
|
||||
final screenTrack = screenStream.getVideoTracks().first;
|
||||
final videoSender = pc.getSenders()
|
||||
.firstWhere((s) => s.track?.kind == 'video');
|
||||
await videoSender.replaceTrack(screenTrack);
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Pyramid VoIP-Architektur
|
||||
|
||||
```
|
||||
Pyramid Call-Stack:
|
||||
┌─────────────────────────────────────────────────────┐
|
||||
│ Dart (Flutter) │
|
||||
│ lib/features/call/ │
|
||||
│ call_notifier.dart — Zustand, Aktionen │
|
||||
│ call_page.dart — UI (Kamera/Mic Controls) │
|
||||
│ livekit_service.dart — LiveKit-Token-Tausch │
|
||||
├─────────────────────────────────────────────────────┤
|
||||
│ Matrix SDK │
|
||||
│ room.sendState(com.famedly.call.member) │
|
||||
│ client.sendToDevice(com.famedly.call) │
|
||||
│ client.requestOpenIdToken() │
|
||||
├─────────────────────────────────────────────────────┤
|
||||
│ LiveKit Client (livekit_client package) │
|
||||
│ LiveKitClient.connect(url, token) │
|
||||
│ room.localParticipant.setMicrophoneEnabled() │
|
||||
│ VideoTrackRenderer Widget │
|
||||
├─────────────────────────────────────────────────────┤
|
||||
│ Native (Android/iOS) │
|
||||
│ FlutterForegroundService — hält Call am Leben │
|
||||
│ Klingelton via AudioManager │
|
||||
└─────────────────────────────────────────────────────┘
|
||||
|
||||
TURN/STUN-Server Konfiguration (from Matrix):
|
||||
await client.getTurnServer() →
|
||||
TurnServerCredentials(
|
||||
username, password, ttl,
|
||||
uris: ['turn:turn.example.org:5349?transport=udp']
|
||||
)
|
||||
```
|
||||
|
||||
```dart
|
||||
// TURN-Credentials vom Homeserver holen
|
||||
final turn = await client.getTurnServer();
|
||||
final iceServers = [
|
||||
{'urls': turn.uris, 'username': turn.username, 'credential': turn.password},
|
||||
{'urls': 'stun:stun.matrix.org:3478'},
|
||||
];
|
||||
```
|
||||
Reference in New Issue
Block a user