From 8133919e35c6c6fab16f21366e55274bde7c65eb Mon Sep 17 00:00:00 2001 From: Peter Thatcher <pthatcher@microsoft.com> Date: Tue, 9 Jul 2024 07:52:56 -0600 Subject: [PATCH 1/3] Add use case 3 for custom NACK/RTX --- api-outline.md | 14 ++++++++++- explainer-use-case-1.md | 4 +-- explainer-use-case-2.md | 6 ++--- explainer-use-case-3.md | 54 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 72 insertions(+), 6 deletions(-) create mode 100644 explainer-use-case-3.md diff --git a/api-outline.md b/api-outline.md index e5f521e..ea0d323 100644 --- a/api-outline.md +++ b/api-outline.md @@ -124,6 +124,9 @@ interface RTCRtpSendStream { attribute EventHandler onpacketizedrtp; sequence<RTCRtpPacket> readPacketizedRtp(long maxNumberOfPackets); + attribute EventHandler onreceivedrtcpnacks; + sequence<RTCRtpNack> readReceivedRtcpNacks(long maxNumberOfPackets); + // https://github.com/w3c/webrtc-rtptransport/issues/32 void sendRtp(RTCRtpPacket packet); Promise<RTCRtpSendResult> sendRtp(RTCRtpPacketInit packet, RTCRtpSendOptions options); @@ -152,6 +155,13 @@ enum RTCRtpUnsentReason { dictionary RTCRtpSendOptions { DOMHighResTimeStamp sendTime; + // Serializes the first two bytes of the RTP payload as an RTX payload + // and sends using the RTX ssrc. + bool asRtx; +} + +interface RTCRtcpNack { + readonly attribute sequence<unsigned short> sequenceNumbers; } [Exposed=(Window,Worker), Transferable] @@ -164,6 +174,8 @@ interface RTCRtpReceiveStream { attribute EventHandler onreceivedrtp; sequence<RTCRtpPacket> readReceivedRtp(long maxNumberOfPackets); - void receiveRtp(RTCRtpPacket packet) + void receiveRtp(RTCRtpPacket packet); + + void sendNack(sequence<unsigned short>); } ``` \ No newline at end of file diff --git a/explainer-use-case-1.md b/explainer-use-case-1.md index 9c50af4..09acb75 100644 --- a/explainer-use-case-1.md +++ b/explainer-use-case-1.md @@ -260,7 +260,7 @@ setInterval(() => { }, 1000); ``` -## Example 13: Receive with BYOB +### Example 13: Receive with BYOB ```javascript const [pc, videoRtpReceiver] = await setupPeerConnectionWithRtpReceiver(); // Custom const videoRtpReceiveStream = await videoRtpReceiver.replaceReceiveStreams()[0]; // Custom @@ -274,7 +274,7 @@ videoRtpReceiveStream.onrtpreceived = () => { }; ``` -## Example 14: Packetize with BYOB +### Example 14: Packetize with BYOB ```javascript const [pc, videoRtpSender] = await setupPeerConnectionWithRtpSender(); // Custom const videoRtpSendStream = await videoRtpSender.replaceSendStreams()[0]; diff --git a/explainer-use-case-2.md b/explainer-use-case-2.md index 64eff05..e9b4db1 100644 --- a/explainer-use-case-2.md +++ b/explainer-use-case-2.md @@ -21,7 +21,7 @@ Applications can do custom bandwidth estimation via: ## Examples -## Example 1: Custom BWE +### Example 1: Custom BWE ```javascript const [pc, rtpTransport] = setupPeerConnectionWithRtpTransport(); // Custom @@ -40,7 +40,7 @@ rtpTransport.onrtpacksreceived = (rtpAcks) => { ``` -## Example 2: Custom Pacing and Probing +### Example 2: Custom Pacing and Probing ```javascript const [pc, rtpTransport] = setupPeerConnectionWithRtpTransport({customPacer: true}); // Custom @@ -59,7 +59,7 @@ while (true) { } ``` -## Example 3: Batched pacing +### Example 3: Batched pacing Making use of the synchronous readPacketizedRtp method to only read packets in batches at a controlled frequency. diff --git a/explainer-use-case-3.md b/explainer-use-case-3.md new file mode 100644 index 0000000..c15ed9a --- /dev/null +++ b/explainer-use-case-3.md @@ -0,0 +1,54 @@ +# Custom RTX Use Case + +## Motivation + +Advanced web applications wish to control NACK and RTX behavior, such as being able to control how large the RTX packet cache is, or how long packets stay in the cache. + +## Examples + +### Example: Custom RTX packet cache + +```javascript +const [pc, rtpTransport, rtpSender] = setupPeerConnectionWithRtpTransport(); // Custom +const rtxPacketCache = createRtxPacketCache(); // Custom +const rtpSendStream = await rtpSender.replaceSendStreams()[0]; +rtpSendStream.onpacketizedrtp = () => { + const rtpPackets = rtpSendStream.readPacketizedRtp(); + for (const rtpPacket of rtpPackets) { + rtxPacketCache.cacheRtpPacket(rtpPacket); + rtpSendStream.sendRtp(rtpPacket); + } +}; +// TODO: How do we disable normal NACK processing? +rtpSendStream.onreceivedrtcpnacks = () => { + const nacks = rtpSendStream.readReceivedRtcpNacks(); + for (const nack of nacks) { + for (const seqnum of nack.sequenceNumbers) { + const cachedRtpPacket = rtxPacketCache.getCachedRtpPacketBySequenceNumber(seqnum); + if (cachedRtpPacket) { + rtpSendStream.sendRtp(cachedRtpPacket, {asRtx: true}); + } + } + } +} +``` + +### Example: Custom NACK + +```javascript +const [pc, rtpTransport, rtpReceiver] = setupPeerConnectionWithRtpTransport(); // Custom +const nackCalculator = createNackCalculator(); // Custom +const rtpReceiveStream = await rtpReceiver.replaceReceiveStreams()[0]; +// TODO: How do we disable normal NACK sending? +rtpReceiveStream.onrtpreceived = () => { + const rtpPacket = rtpReceiveStream.readReceivedRtp(); + const nackedSequenceNumbers = nackCalaculator.calculateNackedSequenceNumbers(rtpPacket); + if (nackedSequenceNumbers) { + rtpRecieveStream.sendNack(nackedSequenceNumbers); + } +} +``` + +### Example: Custom RTX payload + +TODO \ No newline at end of file From bd97d898eea746aec0a5365ba2fa030d73e08907 Mon Sep 17 00:00:00 2001 From: Peter Thatcher <pthatcher@microsoft.com> Date: Tue, 23 Jul 2024 07:43:36 -0600 Subject: [PATCH 2/3] Add variants of the examples for cross-transpor and unparsed --- explainer-use-case-3.md | 117 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 113 insertions(+), 4 deletions(-) diff --git a/explainer-use-case-3.md b/explainer-use-case-3.md index c15ed9a..059c586 100644 --- a/explainer-use-case-3.md +++ b/explainer-use-case-3.md @@ -6,7 +6,7 @@ Advanced web applications wish to control NACK and RTX behavior, such as being a ## Examples -### Example: Custom RTX packet cache +### Example 1a: Custom RTX packet cache (per-stream and parsed) ```javascript const [pc, rtpTransport, rtpSender] = setupPeerConnectionWithRtpTransport(); // Custom @@ -16,16 +16,19 @@ rtpSendStream.onpacketizedrtp = () => { const rtpPackets = rtpSendStream.readPacketizedRtp(); for (const rtpPacket of rtpPackets) { rtxPacketCache.cacheRtpPacket(rtpPacket); + // CON: Need sendRtp rtpSendStream.sendRtp(rtpPacket); } }; // TODO: How do we disable normal NACK processing? rtpSendStream.onreceivedrtcpnacks = () => { + // NEW: readReceivedRtcpNacks and Nack.sequenceNumbers const nacks = rtpSendStream.readReceivedRtcpNacks(); for (const nack of nacks) { for (const seqnum of nack.sequenceNumbers) { const cachedRtpPacket = rtxPacketCache.getCachedRtpPacketBySequenceNumber(seqnum); if (cachedRtpPacket) { + // NEW: asRtx rtpSendStream.sendRtp(cachedRtpPacket, {asRtx: true}); } } @@ -33,7 +36,72 @@ rtpSendStream.onreceivedrtcpnacks = () => { } ``` -### Example: Custom NACK +### Example 1b: Custom RTX packet cache (cross-RtpTransport and parsed) + +```javascript +const [pc, rtpTransport] = setupPeerConnectionWithRtpTransport(); // Custom +const rtxPacketCache = createRtxPacketCache(); // Custom +rtpTransport.onrtpsent = () => { + // NEW: RtpTransport.readSentRtp (make RtpTransport.onrtpsent batch-style) + const sentRtps = rtpTransport.readSentRtp(); + for (const sentRtp of sentRtps) { + // NEW: SentRtp.packet (make the existing object include the whole RtpPacket, not just timestamps) + rtxPacketCache.cacheRtpPacket(sentRtp.packet); + // PRO: No sendRtp required + } +}; +// TODO: How do we disable normal NACK processing? +// NEW: RtpTransport.onreceivedrtcp (a whole new thing) +rtpTransport.onreceivedrtcp = () => { + // NEW: RtpTransport.readReceivedRtcp (batched part of RtpTransport.onrecievedrtcp) + // NEW: RtcpPacket.nacks and Nack.sequenceNumbers and Nack.ssrc + const rtcp = rtpTransport.readReceivedRtcp(); + for (const nack of rtcp.nacks) { + for (const seqnum of nack.sequenceNumbers) { + const cachedRtpPacket = rtxPacketCache.getCachedRtpPacket(nack.ssrc, seqnum); + if (cachedRtpPacket) { + // NEW: asRtx + rtpSendStream.sendRtp(cachedRtpPacket, {asRtx: true}); + } + } + } +} +``` + +### Example 1c: Custom RTX packet cache (cross-RtpTransport and unparsed) + +```javascript +const [pc, rtpTransport] = setupPeerConnectionWithRtpTransport(); // Custom +const rtxPacketCache = createRtxPacketCache(); // Custom + // NEW: RtpTransport.readSentRtp (make RtpTransport.onrtpsent batch-style) + const sentRtps = rtpTransport.readSentRtp(); + for (const sentRtp of sentRtps) { + // NEW: SentRtp.packet (make the existing object include the whole RtpPacket, not just timestamps) + rtxPacketCache.cacheRtpPacket(sentRtp.packet); + // PRO: No sendRtp required + } +}; +// TODO: How do we disable normal NACK processing? +// NEW: RtpTransport.onreceivedrtcp (a whole new thing) +rtpTransport.onreceivedrtcp = () => { + // NEW: RtpTransport.readReceivedRtcp (batched part of RtpTransport.onrecievedrtcp) + const rtcp = rtpTransport.readReceivedRtcp(); + // PRO: No need for browser to parse NACKs + // CON: Need to parse NACKs yourself + const nacks = parseNacks(rtcp); + for (const nack of nacks) { + for (const seqnum of nack.sequenceNumbers) { + const cachedRtpPacket = rtxPacketCache.getCachedRtpPacket(nack.ssrc, seqnum); + if (cachedRtpPacket) { + // NEW: asRtx + rtpSendStream.sendRtp(cachedRtpPacket, {asRtx: true}); + } + } + } +} +``` + +### Example 2a: Custom NACK (per-stream and "parsed") ```javascript const [pc, rtpTransport, rtpReceiver] = setupPeerConnectionWithRtpTransport(); // Custom @@ -41,14 +109,55 @@ const nackCalculator = createNackCalculator(); // Custom const rtpReceiveStream = await rtpReceiver.replaceReceiveStreams()[0]; // TODO: How do we disable normal NACK sending? rtpReceiveStream.onrtpreceived = () => { - const rtpPacket = rtpReceiveStream.readReceivedRtp(); - const nackedSequenceNumbers = nackCalaculator.calculateNackedSequenceNumbers(rtpPacket); + const rtpPackets = rtpReceiveStream.readReceivedRtp(); + const nackedSequenceNumbers = nackCalaculator.calculateNackedSequenceNumbers(rtpPackets); if (nackedSequenceNumbers) { rtpRecieveStream.sendNack(nackedSequenceNumbers); } } ``` +### Example 2b: Custom NACK (per-transport and "parsed") + +```javascript +const [pc, rtpTransport] = setupPeerConnectionWithRtpTransport(); // Custom +const nackCalculator = createNackCalculator(); // Custom +// TODO: How do we disable normal NACK sending? +// NEW: RtpTransport.onrtpreceived (transport-wide version of RtpReceiveStream.onrtpreceived) +// CON: Duplicate onrtpreceived (transport-wide and per-stream) +rtpTransport.onrtpreceived = () => { + // NEW: RtpTransport.readReceivedRtp (transport-wide version of RtpReceiveStream.readReceivedRtp) + const rtpPackets = rtpTransport.readReceivedRtp(); + const nacks = nackCalaculator.calculateNacks(rtpPackets); + for (const nack of nacks) { + // NEW: RtpTransport.sendNack (or maybe sendRtcp) + rtpTransport.sendNack(nack.ssrc, nack.sequenceNumbers); + } +} +``` + +### Example 2c: Custom NACK (per-transport and "unparsed") + +```javascript +const [pc, rtpTransport] = setupPeerConnectionWithRtpTransport(); // Custom +const nackCalculator = createNackCalculator(); // Custom +// TODO: How do we disable normal NACK sending? +// NEW: RtpTransport.onrtpreceived (transport-wide version of RtpReceiveStream.onrtpreceived) +// CON: Duplicate onrtpreceived (transport-wide and per-stream) +rtpTransport.onrtpreceived = () => { + // NEW: RtpTransport.readReceivedRtp (transport-wide version of RtpReceiveStream.readReceivedRtp) + const rtpPackets = rtpTransport.readReceivedRtp(); + const nacks = nackCalaculator.calculateNacks(rtpPackets); + for (const nack of nacks) { + // PRO: Browser doesn't have to construct NACK + // CON: User does have to construct NACK + const rtcp = constructNackPacket(nack.ssrc, nack.sequenceNumbers); + rtpTransport.sendRtcp(rtcp); + } +} +``` + + ### Example: Custom RTX payload TODO \ No newline at end of file From bb5d4a7a7a6fbbce4bbb8a47fa88ce98b083aa4d Mon Sep 17 00:00:00 2001 From: Peter <pthatcher@microsoft.com> Date: Tue, 20 Aug 2024 07:47:49 -0600 Subject: [PATCH 3/3] Add cross-transport customNack and onpacketizedrtcpavailable/readPacketizedRtcp --- api-outline.md | 13 +- explainer-use-case-3.md | 323 ++++++++++++++++++++-------------------- 2 files changed, 172 insertions(+), 164 deletions(-) diff --git a/api-outline.md b/api-outline.md index ea0d323..58f1880 100644 --- a/api-outline.md +++ b/api-outline.md @@ -44,6 +44,10 @@ dictionary RTCRtpHeaderExtensionInit { required AllowSharedBufferSource value; } +interface RTCRtcpPacket { + readonly attribute sequence<RTCRtcpNack> nacks; +} + ``` ### RTCPeerConnection, RTCRtpSendStream, RTCRtpReceiveStream Extensions @@ -60,6 +64,11 @@ dictionary RTCConfiguration { // and I will send them." // TODO: Think of a better name bool customPacer; + // Means "continue to encode and packetize RTCP NACK, but don't send them. + // Instead give them to me via onpacketizedrtcpavailable/readPacketizedRtcp + // and I will send them." + // TODO: Think of a better name + bool customNack; } partial interface RTCRtpSender { @@ -81,6 +90,8 @@ interface RTCRtpTransport { attribute EventHandler onrtpacksreceived; // RtpAcks attribute EventHandler onpacketizedrtpavailable; // No payload. Call readPacketizedRtp sequence<RTCRtpPacket> readPacketizedRtp(maxNumberOfPackets); + attribute EventHandler onpacketizedrtcpavailable; // No payload. Call readPacketizedRtcp + sequence<RTCRtcpPacket> readPacketizedRtcp(maxNumberOfPackets); readonly attribute unsigned long bandwidthEstimate; // bps readonly attribute unsigned long allocatedBandwidth; // bps @@ -125,7 +136,7 @@ interface RTCRtpSendStream { sequence<RTCRtpPacket> readPacketizedRtp(long maxNumberOfPackets); attribute EventHandler onreceivedrtcpnacks; - sequence<RTCRtpNack> readReceivedRtcpNacks(long maxNumberOfPackets); + sequence<RTCRtcpNack> readReceivedRtcpNacks(long maxNumberOfPackets); // https://github.com/w3c/webrtc-rtptransport/issues/32 void sendRtp(RTCRtpPacket packet); diff --git a/explainer-use-case-3.md b/explainer-use-case-3.md index 059c586..2b185f3 100644 --- a/explainer-use-case-3.md +++ b/explainer-use-case-3.md @@ -1,163 +1,160 @@ -# Custom RTX Use Case - -## Motivation - -Advanced web applications wish to control NACK and RTX behavior, such as being able to control how large the RTX packet cache is, or how long packets stay in the cache. - -## Examples - -### Example 1a: Custom RTX packet cache (per-stream and parsed) - -```javascript -const [pc, rtpTransport, rtpSender] = setupPeerConnectionWithRtpTransport(); // Custom -const rtxPacketCache = createRtxPacketCache(); // Custom -const rtpSendStream = await rtpSender.replaceSendStreams()[0]; -rtpSendStream.onpacketizedrtp = () => { - const rtpPackets = rtpSendStream.readPacketizedRtp(); - for (const rtpPacket of rtpPackets) { - rtxPacketCache.cacheRtpPacket(rtpPacket); - // CON: Need sendRtp - rtpSendStream.sendRtp(rtpPacket); - } -}; -// TODO: How do we disable normal NACK processing? -rtpSendStream.onreceivedrtcpnacks = () => { - // NEW: readReceivedRtcpNacks and Nack.sequenceNumbers - const nacks = rtpSendStream.readReceivedRtcpNacks(); - for (const nack of nacks) { - for (const seqnum of nack.sequenceNumbers) { - const cachedRtpPacket = rtxPacketCache.getCachedRtpPacketBySequenceNumber(seqnum); - if (cachedRtpPacket) { - // NEW: asRtx - rtpSendStream.sendRtp(cachedRtpPacket, {asRtx: true}); - } - } - } -} -``` - -### Example 1b: Custom RTX packet cache (cross-RtpTransport and parsed) - -```javascript -const [pc, rtpTransport] = setupPeerConnectionWithRtpTransport(); // Custom -const rtxPacketCache = createRtxPacketCache(); // Custom -rtpTransport.onrtpsent = () => { - // NEW: RtpTransport.readSentRtp (make RtpTransport.onrtpsent batch-style) - const sentRtps = rtpTransport.readSentRtp(); - for (const sentRtp of sentRtps) { - // NEW: SentRtp.packet (make the existing object include the whole RtpPacket, not just timestamps) - rtxPacketCache.cacheRtpPacket(sentRtp.packet); - // PRO: No sendRtp required - } -}; -// TODO: How do we disable normal NACK processing? -// NEW: RtpTransport.onreceivedrtcp (a whole new thing) -rtpTransport.onreceivedrtcp = () => { - // NEW: RtpTransport.readReceivedRtcp (batched part of RtpTransport.onrecievedrtcp) - // NEW: RtcpPacket.nacks and Nack.sequenceNumbers and Nack.ssrc - const rtcp = rtpTransport.readReceivedRtcp(); - for (const nack of rtcp.nacks) { - for (const seqnum of nack.sequenceNumbers) { - const cachedRtpPacket = rtxPacketCache.getCachedRtpPacket(nack.ssrc, seqnum); - if (cachedRtpPacket) { - // NEW: asRtx - rtpSendStream.sendRtp(cachedRtpPacket, {asRtx: true}); - } - } - } -} -``` - -### Example 1c: Custom RTX packet cache (cross-RtpTransport and unparsed) - -```javascript -const [pc, rtpTransport] = setupPeerConnectionWithRtpTransport(); // Custom -const rtxPacketCache = createRtxPacketCache(); // Custom - // NEW: RtpTransport.readSentRtp (make RtpTransport.onrtpsent batch-style) - const sentRtps = rtpTransport.readSentRtp(); - for (const sentRtp of sentRtps) { - // NEW: SentRtp.packet (make the existing object include the whole RtpPacket, not just timestamps) - rtxPacketCache.cacheRtpPacket(sentRtp.packet); - // PRO: No sendRtp required - } -}; -// TODO: How do we disable normal NACK processing? -// NEW: RtpTransport.onreceivedrtcp (a whole new thing) -rtpTransport.onreceivedrtcp = () => { - // NEW: RtpTransport.readReceivedRtcp (batched part of RtpTransport.onrecievedrtcp) - const rtcp = rtpTransport.readReceivedRtcp(); - // PRO: No need for browser to parse NACKs - // CON: Need to parse NACKs yourself - const nacks = parseNacks(rtcp); - for (const nack of nacks) { - for (const seqnum of nack.sequenceNumbers) { - const cachedRtpPacket = rtxPacketCache.getCachedRtpPacket(nack.ssrc, seqnum); - if (cachedRtpPacket) { - // NEW: asRtx - rtpSendStream.sendRtp(cachedRtpPacket, {asRtx: true}); - } - } - } -} -``` - -### Example 2a: Custom NACK (per-stream and "parsed") - -```javascript -const [pc, rtpTransport, rtpReceiver] = setupPeerConnectionWithRtpTransport(); // Custom -const nackCalculator = createNackCalculator(); // Custom -const rtpReceiveStream = await rtpReceiver.replaceReceiveStreams()[0]; -// TODO: How do we disable normal NACK sending? -rtpReceiveStream.onrtpreceived = () => { - const rtpPackets = rtpReceiveStream.readReceivedRtp(); - const nackedSequenceNumbers = nackCalaculator.calculateNackedSequenceNumbers(rtpPackets); - if (nackedSequenceNumbers) { - rtpRecieveStream.sendNack(nackedSequenceNumbers); - } -} -``` - -### Example 2b: Custom NACK (per-transport and "parsed") - -```javascript -const [pc, rtpTransport] = setupPeerConnectionWithRtpTransport(); // Custom -const nackCalculator = createNackCalculator(); // Custom -// TODO: How do we disable normal NACK sending? -// NEW: RtpTransport.onrtpreceived (transport-wide version of RtpReceiveStream.onrtpreceived) -// CON: Duplicate onrtpreceived (transport-wide and per-stream) -rtpTransport.onrtpreceived = () => { - // NEW: RtpTransport.readReceivedRtp (transport-wide version of RtpReceiveStream.readReceivedRtp) - const rtpPackets = rtpTransport.readReceivedRtp(); - const nacks = nackCalaculator.calculateNacks(rtpPackets); - for (const nack of nacks) { - // NEW: RtpTransport.sendNack (or maybe sendRtcp) - rtpTransport.sendNack(nack.ssrc, nack.sequenceNumbers); - } -} -``` - -### Example 2c: Custom NACK (per-transport and "unparsed") - -```javascript -const [pc, rtpTransport] = setupPeerConnectionWithRtpTransport(); // Custom -const nackCalculator = createNackCalculator(); // Custom -// TODO: How do we disable normal NACK sending? -// NEW: RtpTransport.onrtpreceived (transport-wide version of RtpReceiveStream.onrtpreceived) -// CON: Duplicate onrtpreceived (transport-wide and per-stream) -rtpTransport.onrtpreceived = () => { - // NEW: RtpTransport.readReceivedRtp (transport-wide version of RtpReceiveStream.readReceivedRtp) - const rtpPackets = rtpTransport.readReceivedRtp(); - const nacks = nackCalaculator.calculateNacks(rtpPackets); - for (const nack of nacks) { - // PRO: Browser doesn't have to construct NACK - // CON: User does have to construct NACK - const rtcp = constructNackPacket(nack.ssrc, nack.sequenceNumbers); - rtpTransport.sendRtcp(rtcp); - } -} -``` - - -### Example: Custom RTX payload - -TODO \ No newline at end of file +# Custom RTX Use Case + +## Motivation + +Advanced web applications wish to control NACK and RTX behavior, such as being able to control how large the RTX packet cache is, or how long packets stay in the cache. + +## Examples + +### Example 1a: Custom RTX packet cache (per-stream and parsed) + +```javascript +const [pc, rtpTransport, rtpSender] = setupPeerConnectionWithRtpTransport(); // Custom +const rtxPacketCache = createRtxPacketCache(); // Custom +const rtpSendStream = await rtpSender.replaceSendStreams()[0]; +rtpSendStream.onpacketizedrtp = () => { + const rtpPackets = rtpSendStream.readPacketizedRtp(); + for (const rtpPacket of rtpPackets) { + rtxPacketCache.cacheRtpPacket(rtpPacket); + // CON: Need sendRtp + rtpSendStream.sendRtp(rtpPacket); + } +}; +// TODO: How do we disable normal NACK processing? +rtpSendStream.onreceivedrtcpnacks = () => { + // NEW: readReceivedRtcpNacks and Nack.sequenceNumbers + const nacks = rtpSendStream.readReceivedRtcpNacks(); + for (const nack of nacks) { + for (const seqnum of nack.sequenceNumbers) { + const cachedRtpPacket = rtxPacketCache.getCachedRtpPacketBySequenceNumber(seqnum); + if (cachedRtpPacket) { + // NEW: asRtx + rtpSendStream.sendRtp(cachedRtpPacket, {asRtx: true}); + } + } + } +} +``` + +### Example 1b: Custom RTX packet cache (cross-RtpTransport and parsed) + +```javascript +const [pc, rtpTransport] = setupPeerConnectionWithRtpTransport(); // Custom +const rtxPacketCache = createRtxPacketCache(); // Custom +rtpTransport.onrtpsent = () => { + // NEW: RtpTransport.readSentRtp (make RtpTransport.onrtpsent batch-style) + const sentRtps = rtpTransport.readSentRtp(); + for (const sentRtp of sentRtps) { + // NEW: SentRtp.packet (make the existing object include the whole RtpPacket, not just timestamps) + rtxPacketCache.cacheRtpPacket(sentRtp.packet); + // PRO: No sendRtp required + } +}; +// TODO: How do we disable normal NACK processing? +// NEW: RtpTransport.onreceivedrtcp (a whole new thing) +rtpTransport.onreceivedrtcp = () => { + // NEW: RtpTransport.readReceivedRtcp (batched part of RtpTransport.onrecievedrtcp) + // NEW: RtcpPacket.nacks and Nack.sequenceNumbers and Nack.ssrc + const rtcp = rtpTransport.readReceivedRtcp(); + for (const nack of rtcp.nacks) { + for (const seqnum of nack.sequenceNumbers) { + const cachedRtpPacket = rtxPacketCache.getCachedRtpPacket(nack.ssrc, seqnum); + if (cachedRtpPacket) { + // NEW: asRtx + rtpSendStream.sendRtp(cachedRtpPacket, {asRtx: true}); + } + } + } +} +``` + +### Example 1c: Custom RTX packet cache (cross-RtpTransport and unparsed) + +```javascript +const [pc, rtpTransport] = setupPeerConnectionWithRtpTransport(); // Custom +const rtxPacketCache = createRtxPacketCache(); // Custom + // NEW: RtpTransport.readSentRtp (make RtpTransport.onrtpsent batch-style) + const sentRtps = rtpTransport.readSentRtp(); + for (const sentRtp of sentRtps) { + // NEW: SentRtp.packet (make the existing object include the whole RtpPacket, not just timestamps) + rtxPacketCache.cacheRtpPacket(sentRtp.packet); + // PRO: No sendRtp required + } +}; +// TODO: How do we disable normal NACK processing? +// NEW: RtpTransport.onreceivedrtcp (a whole new thing) +rtpTransport.onreceivedrtcp = () => { + // NEW: RtpTransport.readReceivedRtcp (batched part of RtpTransport.onrecievedrtcp) + const rtcp = rtpTransport.readReceivedRtcp(); + // PRO: No need for browser to parse NACKs + // CON: Need to parse NACKs yourself + const nacks = parseNacks(rtcp); + for (const nack of nacks) { + for (const seqnum of nack.sequenceNumbers) { + const cachedRtpPacket = rtxPacketCache.getCachedRtpPacket(nack.ssrc, seqnum); + if (cachedRtpPacket) { + // NEW: asRtx + rtpSendStream.sendRtp(cachedRtpPacket, {asRtx: true}); + } + } + } +} +``` + +### Example 2a: Custom NACK (per-stream and "parsed") + +```javascript +const [pc, rtpTransport, rtpReceiver] = setupPeerConnectionWithRtpTransport(); // Custom using "customNack: true" +const nackCalculator = createNackCalculator(); // Custom +const rtpReceiveStream = await rtpReceiver.replaceReceiveStreams()[0]; +rtpReceiveStream.onrtpreceived = () => { + const rtpPackets = rtpReceiveStream.readReceivedRtp(); + const nackedSequenceNumbers = nackCalaculator.calculateNackedSequenceNumbers(rtpPackets); + if (nackedSequenceNumbers) { + rtpRecieveStream.sendNack(nackedSequenceNumbers); + } +} +``` + +### Example 2b: Custom NACK (per-transport and "parsed") + +```javascript +const [pc, rtpTransport] = setupPeerConnectionWithRtpTransport(); // Custom using "customNack: true" +const nackCalculator = createNackCalculator(); // Custom +// NEW: RtpTransport.onrtpreceived (transport-wide version of RtpReceiveStream.onrtpreceived) +// CON: Duplicate onrtpreceived (transport-wide and per-stream) +rtpTransport.onrtpreceived = () => { + // NEW: RtpTransport.readReceivedRtp (transport-wide version of RtpReceiveStream.readReceivedRtp) + const rtpPackets = rtpTransport.readReceivedRtp(); + const nacks = nackCalaculator.calculateNacks(rtpPackets); + for (const nack of nacks) { + // NEW: RtpTransport.sendNack (or maybe sendRtcp) + rtpTransport.sendNack(nack.ssrc, nack.sequenceNumbers); + } +} +``` + +### Example 2c: Custom NACK (per-transport and "unparsed") + +```javascript +const [pc, rtpTransport] = setupPeerConnectionWithRtpTransport(); // Custom using "customNack: true" +const nackCalculator = createNackCalculator(); // Custom +// NEW: RtpTransport.onrtpreceived (transport-wide version of RtpReceiveStream.onrtpreceived) +// CON: Duplicate onrtpreceived (transport-wide and per-stream) +rtpTransport.onrtpreceived = () => { + // NEW: RtpTransport.readReceivedRtp (transport-wide version of RtpReceiveStream.readReceivedRtp) + const rtpPackets = rtpTransport.readReceivedRtp(); + const nacks = nackCalaculator.calculateNacks(rtpPackets); + for (const nack of nacks) { + // PRO: Browser doesn't have to construct NACK + // CON: User does have to construct NACK + const rtcp = constructNackPacket(nack.ssrc, nack.sequenceNumbers); + rtpTransport.sendRtcp(rtcp); + } +} +``` + + +### Example: Custom RTX payload + +TODO: Use onpacketizedrtcpavailable/readPacketizedRtcp? \ No newline at end of file