使用 aiortc 流式传输实时视频源而不是视频文件

Question

我正在尝试使用 Mss 流式传输桌面的实时信息。

但是我似乎无法实现这一目标并使用主动脉。相反，我只能将视频文件流式传输到我的客户端。这是我正在运行的。

import asyncio
import json
import logging
import os

from aiohttp import web
from aiortc import RTCPeerConnection, RTCSessionDescription
from aiortc.contrib.media import MediaPlayer

ROOT = os.path.dirname(__file__)

def create_local_tracks(play_from):
    player = MediaPlayer(play_from)
    return player.video 


async def index(request):
    content = open(os.path.join(ROOT, "index.html"), "r").read()
    return web.Response(content_type="text/html", text=content)


async def offer(request):
    params = await request.json()
    offer = RTCSessionDescription(sdp=params["sdp"], type=params["type"])
    pc = RTCPeerConnection()
    pcs.add(pc)

    # open media source
    video = create_local_tracks("video.mp4")

    pc.addTrack(video)

    await pc.setRemoteDescription(offer)

    answer = await pc.createAnswer()
    await pc.setLocalDescription(answer)

    return web.Response(
        content_type="application/json",
        text=json.dumps(
            {"sdp": pc.localDescription.sdp, "type": pc.localDescription.type}
        ),
    )

pcs = set()
async def on_shutdown(app):
    coros = [pc.close() for pc in pcs]
    await asyncio.gather(*coros)
    pcs.clear()

if __name__ == "__main__":
    logging.basicConfig(level=logging.INFO)

    app = web.Application()
    app.on_shutdown.append(on_shutdown)
    app.router.add_get("/", index)
    app.router.add_post("/offer", offer)
    web.run_app(app, host="0.0.0.0", port=8080)

现在，当连接时，它会将 video.mp4 的视频流式传输到 html 客户端。这对于流式传输静态文件非常有效，但我无法弄清楚如何流式传输用户屏幕的实时视频。我认为名为 mss 的 python 屏幕记录库适合于此，因为它提供 webRTC 能够处理的高帧率。谢谢！

Answer 1

要流式传输用户的屏幕，您需要修改“create_local_tracks”函数并添加一个新类来捕获用户的屏幕。这是一个例子：

from aiortc import VideoStreamTrack
from av import VideoFrame
import numpy as np
import threading
import asyncio
import queue
import mss

class ScreenCapturing(VideoStreamTrack):
    def __init__(self) -> None:
        super().__init__()
        self.queue = queue.Queue(10)

    async def recv(self):
        img = self.queue.get()
        # Convert RGBA to RGB by discarding the alpha channel
        img_rgb = img[:, :, :3]
        frame = VideoFrame.from_ndarray(img_rgb, format="bgr24")
        pts, time_base = await self.next_timestamp()
        frame.pts = pts
        frame.time_base = time_base
        return frame

    def start(self):
        loop = asyncio.new_event_loop()
        asyncio.set_event_loop(loop)
        with mss.mss() as sct:
            monitor = sct.monitors[1]
            while True:
                im = sct.grab(monitor)
                im_np = np.array(im)
                self.queue.put(im_np)

    def stop(self):
        pass


async def create_local_tracks():
    screencapturing = ScreenCapturing()
    threading.Thread(target=screencapturing.start, daemon=True).start()
    # print(await screencapturing.recv())
    return screencapturing