[我正在尝试建立一个客户端服务器体系结构,在该体系结构中,我正在使用getUserMedia()从用户的网络摄像头捕获实时视频。现在,我不想将视频直接显示在<video>
标签中,而是将其发送到我的烧瓶服务器,对帧进行一些处理,然后将其返回到我的网页。
我已使用socketio创建客户端-服务器连接。这是我的index.html中的脚本。请原谅我的错误或任何错误的代码。
<div id="container">
<video autoplay="true" id="videoElement">
</video>
</div>
<script type="text/javascript" charset="utf-8">
var socket = io('http://127.0.0.1:5000');
// checking for connection
socket.on('connect', function(){
console.log("Connected... ", socket.connected)
});
var video = document.querySelector("#videoElement");
// asking permission to access the system camera of user, capturing live
// video on getting true.
if (navigator.mediaDevices.getUserMedia) {
navigator.mediaDevices.getUserMedia({ video: true })
.then(function (stream) {
// instead of showing it directly in <video>, I want to send these frame to server
//video_t.srcObject = stream
//this code might be wrong, but this is what I want to do.
socket.emit('catch-frame', { image: true, buffer: getFrame() });
})
.catch(function (err0r) {
console.log(err0r)
console.log("Something went wrong!");
});
}
// returns a frame encoded in base64
const getFrame = () => {
const canvas = document.createElement('canvas');
canvas.width = video_t.videoWidth;
canvas.height = video_t.videoHeight;
canvas.getContext('2d').drawImage(video_t, 0, 0);
const data = canvas.toDataURL('image/png');
return data;
}
// receive the frame from the server after processed and now I want display them in either
// <video> or <img>
socket.on('response_back', function(frame){
// this code here is wrong, but again this is what something I want to do.
video.srcObject = frame;
});
</script>
在我的app.py-
中from flask import Flask, render_template
from flask_socketio import SocketIO, emit
app = Flask(__name__)
socketio = SocketIO(app)
@app.route('/', methods=['POST', 'GET'])
def index():
return render_template('index.html')
@socketio.on('catch-frame')
def catch_frame(data):
## getting the data frames
## do some processing
## send it back to client
emit('response_back', data) ## ??
if __name__ == '__main__':
socketio.run(app, host='127.0.0.1')
我也曾想过通过WebRTC来做到这一点,但是我只是得到对等的代码。
所以,有人可以帮助我吗?在此先感谢您的帮助。
所以,我试图做的是获取由客户端的网络摄像头捕获的实时视频流,并在后端对其进行处理。
我的后端代码是用Python编写的,我正在使用SocketIo将帧从前端发送到后端。您可以看一下此设计,以更好地了解正在发生的事情-image
这里是工作代码-
app.py
@socketio.on('image')
def image(data_image):
sbuf = StringIO()
sbuf.write(data_image)
# decode and convert into image
b = io.BytesIO(base64.b64decode(data_image))
pimg = Image.open(b)
# Process the image frame
frame = imutils.resize(frame, width=700)
frame = cv2.flip(frame, 1)
imgencode = cv2.imencode('.jpg', frame)[1]
# base64 encode
stringData = base64.b64encode(imgencode).decode('utf-8')
b64_src = 'data:image/jpg;base64,'
stringData = b64_src + stringData
# emit the frame back
emit('response_back', stringData)
index.html
<div id="container">
<canvas id="canvasOutput"></canvas>
<video autoplay="true" id="videoElement"></video>
</div>
<div class = 'video'>
<img id="image">
</div>
<script>
var socket = io('http://localhost:5000');
socket.on('connect', function(){
console.log("Connected...!", socket.connected)
});
const video = document.querySelector("#videoElement");
video.width = 500;
video.height = 375; ;
if (navigator.mediaDevices.getUserMedia) {
navigator.mediaDevices.getUserMedia({ video: true })
.then(function (stream) {
video.srcObject = stream;
video.play();
})
.catch(function (err0r) {
console.log(err0r)
console.log("Something went wrong!");
});
}
let src = new cv.Mat(video.height, video.width, cv.CV_8UC4);
let dst = new cv.Mat(video.height, video.width, cv.CV_8UC1);
let cap = new cv.VideoCapture(video);
const FPS = 22;
setInterval(() => {
cap.read(src);
var type = "image/png"
var data = document.getElementById("canvasOutput").toDataURL(type);
data = data.replace('data:' + type + ';base64,', ''); //split off junk
at the beginning
socket.emit('image', data);
}, 10000/FPS);
socket.on('response_back', function(image){
const image_id = document.getElementById('image');
image_id.src = image;
});
</script>
而且,websockets在安全来源上运行。