方向是Unity --> JavaScript，这里Unity是抓取视频的。 使用 WebRTC，Unity 和 JavaScript 客户端能够相互通信，但 JavaScript 客户端似乎无法显示 Unity 客户端发送的任何视频。
我不知道为什么尽管有清晰的通信（即接收），JavaScript 客户端仍无法显示任何视频输出。
我已经尝试更改/添加编解码器以现在可用，但我不确定是否需要这样做，因为 docs (Unity WebRTC 2.4.0) 并未真正指定需要编解码器。我正在使用 NativeWebSocket（一个 C# 库）与我的本地服务器通信。 SDP 可以从两个客户端发送，但（再次）似乎没有视频显示在我的 JavaScript 客户端上。
统一客户端
using System.Collections;
using UnityEngine;
using Unity.WebRTC;
using System;
using NativeWebSocket;
public class LocalMediaSender : MonoBehaviour
{
public Camera capture_camera;
private RTCPeerConnection local_connection;
private RTCDataChannel data_channel;
private WebSocket ws;
public string ip = "192.168.X.X"; // TODO: add your IP here
// Start is called before the first frame update
async void Start()
{
ws = new WebSocket($"ws://{ip}:8080");
ws.OnOpen += () =>
{
Debug.Log("Connection open!");
InitializeWebRTC();
};
ws.OnError += (e) =>
{
Debug.Log("Error! " + e);
};
ws.OnClose += (e) =>
{
Debug.Log("Connection closed!");
};
ws.OnMessage += (bytes) =>
{
Debug.Log("OnMessage!");
Debug.Log(bytes);
// Getting the message as a string
var message = System.Text.Encoding.UTF8.GetString(bytes);
Debug.Log("OnMessage! " + message);
// Deserialize the message to SDPMessage
SDPMessage sdpMessage = JsonUtility.FromJson<SDPMessage>(message);
// Check if the message is an answer
if (sdpMessage.type == "answer")
{
// Create a RTCSessionDescription for the answer
RTCSessionDescription answer = new RTCSessionDescription
{
type = RTCSdpType.Answer,
sdp = sdpMessage.sdp
};
// Start the coroutine to set the remote description for the local connection
StartCoroutine(HandleSetRemoteDescription(answer));
}
};
await ws.Connect();
}
private void InitializeWebRTC()
{
// [Step 1] Create Peer Connection
local_connection = new RTCPeerConnection(/*ref config*/);
/* ADD EVENT HANDLERS */
{
// [Step 2] onopen Connection Notification
RTCDataChannelInit init = new RTCDataChannelInit { ordered = true }; // to ensure that data packets are delivered in order makes it reliable
data_channel = local_connection.CreateDataChannel("myChannel", init);
data_channel.OnOpen += () =>
{
Debug.Log("Data Channel Opened -- Unity");
};
// [Step 3] onicecandidate To Print SDP
local_connection.OnIceCandidate = (e) =>
{
string iceCandidateString = JsonUtility.ToJson(e);
// Inside the OnIceCandidate event handler in Unity
ws.SendText(iceCandidateString);
//Debug.Log("[ICE Candidate] " + iceCandidateString);
};
}
// [Step 4] addTrack Video Media To Send
if (capture_camera == null){ Debug.LogError("[ERROR] There is no camera!"); }
MediaStream video_stream_track = capture_camera.CaptureStream(1280, 720);
foreach (var track in video_stream_track.GetTracks()) {
local_connection.AddTrack(track, video_stream_track);
}
// [Step 5] createOffer
// use button to start offer
}
public void StartOffer() {
StartCoroutine(HandleCreateOffer());
}
public void StopOffer() {
StopCoroutine(HandleCreateOffer());
}
IEnumerator HandleCreateOffer()
{
RTCOfferAnswerOptions offerOptions = default;
var op = local_connection.CreateOffer(ref offerOptions);
yield return op;
if (op.IsError)
{
Debug.LogError($"Error creating offer: {op.Error}");
yield break;
}
RTCSessionDescription offer = op.Desc;
offer.sdp = offer.sdp.Replace("m=video 9 UDP/TLS/RTP/SAVPF 96 97 98 99 100 101 102", "m=video 9 UDP/TLS/RTP/SAVPF 96");
// [Step 6] setLocalDescription Once Offer Is Successful
var setLocalDescOp = local_connection.SetLocalDescription(ref offer);
yield return setLocalDescOp;
if (setLocalDescOp.IsError)
{
Debug.LogError($"Error setting local description: {setLocalDescOp.Error}");
}
else
{
// Send local_description after it has been set
if (ws.State == WebSocketState.Open)
{
//string sdp_string = JsonUtility.ToJson(local_connection.LocalDescription);
string sdp_string = JsonUtility.ToJson(new SDPMessage { sdp = local_connection.LocalDescription.sdp, type = local_connection.LocalDescription.type.ToString() });
ws.SendText(sdp_string);
Debug.Log("[SDP] " + sdp_string);
}
else
{
Debug.LogError("WebSocket connection is not open.");
}
}
}
IEnumerator HandleSetRemoteDescription(RTCSessionDescription answer)
{
var op = local_connection.SetRemoteDescription(ref answer);
yield return op;
if (op.IsError)
{
Debug.LogError($"Error setting remote description: {op.Error}");
}
else
{
Debug.Log("Remote description set successfully.");
}
}
[Serializable]
public class SDPMessage
{
public string sdp;
public string type;
}
// Update is called once per frame
void Update()
{
// required to recieve messages from signaling server
#if !UNITY_WEBGL || UNITY_EDITOR
ws.DispatchMessageQueue();
#endif
}
}
JavaScript 客户端
<html>
<head>
<title>Local WebRTC Receiver</title>
</head>
<body>
<!-- Create a video element to display the remote video stream -->
<video
style="box-sizing: border-box; border: 5px solid black;"
id="remote-video"
autoplay
playsinline
width="1280"
height="720">
<!-- video body -->
</video>
<script>
let ip = "192.168.X.X" // TODO: add your IP here
const local_connection = new RTCPeerConnection();
let offer = null;
// Connect to the WebSocket signaling server
const signalingServer = new WebSocket(`ws://${ip}:8080`);
signalingServer.onopen = (e) =>{ /* when connected */ }
signalingServer.onmessage = (event) => {
event.data.text().then((msg)=>{
console.log("De-Blobed Message:", msg);
const message = JSON.parse(msg);
console.log("JSON Message:", message);
// Answer offer
if(message.type == "Offer"){
offer = message;
offer.type = "offer"; // <-- THIS IS DUMB BUT NEEDED
local_connection.setRemoteDescription(new RTCSessionDescription(offer))
.then((e)=>{
console.log("offer set!");
return local_connection.createAnswer();
})
.then((answer)=>{
console.log("setting local description!");
return local_connection.setLocalDescription(answer);
})
.then((e)=>{
console.log("Sending Local Description")
const sdp = local_connection.localDescription;
signalingServer.send(JSON.stringify(sdp));
})
.catch((err)=>{console.error(err);})
}
// Handle incoming messages (ICE candidates, offers, etc.)
})
};
local_connection.onicecandidate = (e) =>{
console.log("[SDP] ", local_connection.localDescription);
};
local_connection.ontrack = (e) =>{
console.log("Received remote track:", e.track);
console.log("Received remote streams:", e.streams);
console.log("Kind:", e.track.kind);
if(e.track.kind == "video"){
// Display the remote video stream in the video element
const remoteVideo = document.getElementById('remote-video');
remoteVideo.srcObject = e.streams[0];
console.log(remoteVideo.srcObject);
console.log("Codec: ",e.track.getSettings().codec);
console.log("Settings: ",e.track.getSettings());
}
};
local_connection.ondatachannel = (e) =>{
var receiveChannel = ev.channel;
receiveChannel.onmessage = (e) =>{
console.log("[Message from Unity]\n",e.data);
};
receiveChannel.onopen = (e) =>{
console.log("Connected! -- JS");
};
receiveChannel.onclose = (e) =>{ /* when connection closes*/ };
}
</script>
</body>
</html>
节点 JS 服务器
const WebSocket = require("ws");
const host_ip = "192.168.X.X" // TODO: add your IP here
const server = new WebSocket.Server({ host: host_ip, port: 8080 });
const clients = new Set();
server.on("connection", (socket) => {
clients.add(socket);
console.log("[Server] Client connected");
socket.on("message", (message) => {
console.log("-=-=-=-=-=-[New Message Received]-=-=-=-=-=-");
console.log(message.toString())
console.log("-=-=-=-=-=-[END Message Received]-=-=-=-=-=-");
// console.log("Received:", message);
// Broadcast the message to all other clients
for (const client of clients) {
if (client !== socket) {
client.send(message);
}
}
});
socket.on("close", () => {
clients.delete(socket);
console.log("[Server] Client disconnected");
});
});
console.log(`[Server] WebSocket signaling server running on ws://${host_ip}:8080`);
要运行服务器，请确保安装了 Node.JS，然后运行
npm inpm start