pyramid/docs/matrix-sdk/08-voip.md

08 — VoIP, Calls & LiveKit (MatrixRTC)

---

Inhaltsverzeichnis

• Architektur-Überblick
• 1:1 Call (Legacy m.call)
• MatrixRTC (Gruppen-Calls)
• LiveKit-Integration
• Klingel-Flow
• WebRTC Setup in Flutter
• Call-State Management
• Pyramid VoIP-Architektur

---

Architektur-Überblick

Matrix Signaling (to-device / room events)
        │
        ▼
  MatrixRTC Layer
  ┌─────────────────────────────────┐
  │  com.famedly.call.member Events │
  │  (Membership, Backend, Expiry)  │
  └──────────────┬──────────────────┘
                 │
        ┌────────┴────────┐
        │                 │
    Mesh (P2P)       LiveKit (SFU)
    WebRTC direct    livekit-server
    (1:1, max ~4)    (skalierbar, n:m)

---

1:1 Call (Legacy m.call)

// ── Anruf starten ────────────────────────────────────────────────────────
// Wird vom SDK über room events (m.call.invite, m.call.candidates,
// m.call.answer, m.call.hangup) signalisiert.

// Anruf-Einladung senden (nach WebRTC offer)
await room.sendEvent(
  {
    'call_id': 'eindeutige-call-id',
    'version': 1,
    'lifetime': 60000,              // ms, wie lange das Invite gültig ist
    'offer': {
      'type': 'offer',
      'sdp': sdpString,            // WebRTC SDP Offer
    },
  },
  EventTypes.CallInvite,           // 'm.call.invite'
);

// ICE-Kandidaten senden
await room.sendEvent(
  {
    'call_id': callId,
    'version': 1,
    'candidates': [
      {
        'candidate': 'candidate:...',
        'sdpMid': '0',
        'sdpMLineIndex': 0,
      },
    ],
  },
  EventTypes.CallCandidates,       // 'm.call.candidates'
);

// Anruf annehmen (SDP Answer)
await room.sendEvent(
  {
    'call_id': callId,
    'version': 1,
    'answer': {
      'type': 'answer',
      'sdp': sdpAnswerString,
    },
  },
  EventTypes.CallAnswer,           // 'm.call.answer'
);

// Auflegen
await room.sendEvent(
  {
    'call_id': callId,
    'version': 1,
    'reason': 'user_hangup',        // oder 'ice_failed', 'invite_timeout'
  },
  EventTypes.CallHangup,           // 'm.call.hangup'
);

// Ablehnen (Reject)
await room.sendEvent(
  {
    'call_id': callId,
    'version': 1,
    'reason': 'user_hangup',
  },
  EventTypes.CallReject,           // 'm.call.reject'
);

// Eingehende Anrufe überwachen
client.onCallInvite.stream.listen((CallInvite invite) {
  print('Eingehender Anruf von ${invite.roomId}');
  print('Call-ID: ${invite.callId}');
});

---

MatrixRTC (Gruppen-Calls)

// MatrixRTC verwendet com.famedly.call.member State-Events

// ── Call beitreten (Membership senden) ───────────────────────────────────

final myDeviceId = client.deviceID!;
final callId = room.id;               // Normalerweise = Raum-ID

// Membership-Event senden (signalisiert "ich bin im Call")
await room.client.sendState(
  room.id,
  'org.matrix.msc3401.call.member',  // oder 'com.famedly.call.member'
  {
    'application': 'm.call',
    'call_id': callId,
    'device_id': myDeviceId,
    'expires_ts': DateTime.now()
        .add(const Duration(hours: 1))
        .millisecondsSinceEpoch,
    'backend': {
      'type': 'm.sfu',               // LiveKit
      'livekit_service_url': 'https://livekit.example.org',
    },
    // Für Mesh:
    // 'backend': {'type': 'm.mesh'},
    'membershipID': 'eindeutige-membership-id',
  },
  stateKey: '$myDeviceId:$callId',
);

// ── Call verlassen (Membership entfernen) ─────────────────────────────────

await room.client.sendState(
  room.id,
  'org.matrix.msc3401.call.member',
  {},                                // Leeres Content = Austritt
  stateKey: '$myDeviceId:$callId',
);

// ── Aktive Call-Mitglieder auflesen ──────────────────────────────────────

final callStates = room.states['org.matrix.msc3401.call.member'];
final now = DateTime.now().millisecondsSinceEpoch;
final activeMembers = callStates?.values.where((event) {
  final expiresTs = event.content['expires_ts'] as int? ?? 0;
  return expiresTs > now;           // Nicht abgelaufen
}).toList() ?? [];

for (final member in activeMembers) {
  print(member.content['device_id']);
  print(member.stateKey);           // 'deviceId:callId'
}

---

LiveKit-Integration

// LiveKit-Token vom Matrix-Homeserver anfragen (OpenID)
// Dieser Flow authentifiziert den User beim LiveKit-Server ohne eigenes Passwort.

// 1. OpenID-Token vom Homeserver holen
final openIdToken = await client.requestOpenIdToken(client.userID!);
// openIdToken.accessToken, .matrixServerName

// 2. LiveKit-Token beim MatrixRTC-Service tauschen
// (Pyramid sendet openIdToken an livekit_service_url des Raums)
final response = await http.post(
  Uri.parse('https://livekit-service.example.org/sfu/get'),
  headers: {'Content-Type': 'application/json'},
  body: jsonEncode({
    'room': room.id,
    'openid_token': {
      'access_token': openIdToken.accessToken,
      'token_type': 'Bearer',
      'matrix_server_name': openIdToken.matrixServerName,
    },
    'device_id': client.deviceID,
  }),
);
final livekitToken = jsonDecode(response.body)['token'];

// 3. Mit LiveKit verbinden
import 'package:livekit_client/livekit_client.dart';
final room = await LiveKitClient.connect(
  'wss://livekit.example.org',
  livekitToken,
  roomOptions: const RoomOptions(
    adaptiveStream: true,
    dynacast: true,
    defaultVideoPublishOptions: VideoPublishOptions(
      simulcast: true,
    ),
  ),
);

// 4. Mikrofon aktivieren
await room.localParticipant?.setMicrophoneEnabled(true);

// 5. Kamera aktivieren (Video-Call)
await room.localParticipant?.setCameraEnabled(true);

// 6. Teilnehmer observieren
room.addListener(() {
  for (final participant in room.participants.values) {
    print(participant.identity);    // Matrix User-ID
    for (final track in participant.videoTracks) {
      // Widget: VideoTrackRenderer(track.track!)
    }
  }
});

---

Klingel-Flow

// ── Anruf initiieren ──────────────────────────────────────────────────────

// 1. Membership-Event senden (MatrixRTC)
// 2. To-Device-Event senden für Push-Benachrichtigung beim Empfänger

// Klingelanruf via to-device-Event
await client.sendToDevice(
  'com.famedly.call',              // Event-Typ
  {
    otherUserId: {
      '*': {                       // Alle Geräte des Users
        'call_id': callId,
        'room_id': room.id,
        'type': 'invite',
        'sender_session_id': client.deviceID,
      },
    },
  },
);

// ── Eingehenden Anruf empfangen ───────────────────────────────────────────

client.onToDeviceEvent.stream
    .where((e) => e.type == 'com.famedly.call')
    .listen((event) {
  final callId = event.content['call_id'];
  final roomId = event.content['room_id'];
  final type   = event.content['type']; // 'invite' | 'hangup'
  // Klingelton abspielen, UI anzeigen
});

// ── Anruf ablehnen (während Klingeln) ────────────────────────────────────

await client.sendToDevice(
  'com.famedly.call',
  {
    callerId: {'*': {'call_id': callId, 'type': 'reject'}},
  },
);

---

WebRTC Setup in Flutter

// flutter_webrtc Package
import 'package:flutter_webrtc/flutter_webrtc.dart';

// Media-Stream holen
final stream = await navigator.mediaDevices.getUserMedia({
  'audio': true,
  'video': false,       // true für Video-Call
});

// PeerConnection erstellen
final pc = await createPeerConnection({
  'iceServers': [
    {'urls': 'stun:stun.matrix.org:3478'},
    // TURN-Server (für NAT-Traversal):
    {
      'urls': 'turn:turn.example.org:5349',
      'username': 'user',
      'credential': 'pass',
    },
  ],
  'iceCandidatePoolSize': 10,
});

// Track hinzufügen
for (final track in stream.getTracks()) {
  await pc.addTrack(track, stream);
}

// ICE-Kandidaten sammeln
pc.onIceCandidate = (RTCIceCandidate candidate) {
  // Kandidaten via Matrix senden
  sendIceCandidates([candidate]);
};

// Angebot erstellen
final offer = await pc.createOffer();
await pc.setLocalDescription(offer);
// offer.sdp → via Matrix senden

// Angebot annehmen
await pc.setRemoteDescription(RTCSessionDescription(sdpOffer, 'offer'));
final answer = await pc.createAnswer();
await pc.setLocalDescription(answer);
// answer.sdp → via Matrix senden

// Eingehende Streams
pc.onTrack = (RTCTrackEvent event) {
  final remoteStream = event.streams.first;
  // Widget: RTCVideoRenderer() mit remoteStream
};

// Aufräumen
await pc.close();
await stream.dispose();

---

Call-State Management

// Call-Zustand in Pyramid
enum CallState {
  idle,
  calling,        // Anruf geht raus
  ringing,        // Eingehender Anruf
  connecting,     // ICE/SDP in Bearbeitung
  connected,      // Verbunden
  ended,
}

// Stummschalten
final audioTrack = localStream.getAudioTracks().first;
audioTrack.enabled = false; // Mute
audioTrack.enabled = true;  // Unmute

// Lautsprecher vs. Hörer (Android/iOS)
// flutter_webrtc kümmert sich darum via Helper.setSpeakerphoneOn

// DTMF (Zifferntasten)
final sender = pc.getSenders().firstWhere(
    (s) => s.track?.kind == 'audio');
await sender.dtmf?.insertDTMF('1');

// Screen-Sharing starten
final screenStream = await navigator.mediaDevices.getDisplayMedia({
  'video': true,
  'audio': false,
});
final screenTrack = screenStream.getVideoTracks().first;
final videoSender = pc.getSenders()
    .firstWhere((s) => s.track?.kind == 'video');
await videoSender.replaceTrack(screenTrack);

---

Pyramid VoIP-Architektur

Pyramid Call-Stack:
┌─────────────────────────────────────────────────────┐
│  Dart (Flutter)                                      │
│  lib/features/call/                                  │
│    call_notifier.dart   — Zustand, Aktionen          │
│    call_page.dart       — UI (Kamera/Mic Controls)   │
│    livekit_service.dart — LiveKit-Token-Tausch        │
├─────────────────────────────────────────────────────┤
│  Matrix SDK                                          │
│    room.sendState(com.famedly.call.member)           │
│    client.sendToDevice(com.famedly.call)             │
│    client.requestOpenIdToken()                       │
├─────────────────────────────────────────────────────┤
│  LiveKit Client (livekit_client package)             │
│    LiveKitClient.connect(url, token)                 │
│    room.localParticipant.setMicrophoneEnabled()      │
│    VideoTrackRenderer Widget                         │
├─────────────────────────────────────────────────────┤
│  Native (Android/iOS)                               │
│  FlutterForegroundService — hält Call am Leben       │
│  Klingelton via AudioManager                         │
└─────────────────────────────────────────────────────┘

TURN/STUN-Server Konfiguration (from Matrix):
await client.getTurnServer() →
  TurnServerCredentials(
    username, password, ttl,
    uris: ['turn:turn.example.org:5349?transport=udp']
  )

// TURN-Credentials vom Homeserver holen
final turn = await client.getTurnServer();
final iceServers = [
  {'urls': turn.uris, 'username': turn.username, 'credential': turn.password},
  {'urls': 'stun:stun.matrix.org:3478'},
];