如何获取 HTML 视频的当前帧作为 JavaScript 数组？

Question

按下按钮后，我想将当前视频帧从网络摄像头转换为数组或 tfjs 张量。

我无法找到方法：

• 获取数组
• 将数组裁剪并调整为最大可能的 64x64x3 图像

我想使用/实现一些函数（getFrameFromVideo和crop_to_size），它们可以执行以下操作：

const video = document.getElementById("cam");
var arr_frame;
var cropped_frame;
var tensor_frame;

function crop_to_size(img, xsize, ysize) {
  // crop the image
  return cropped_img;
}

function predict_img() {
  arr_frame = getFrameFromVideo(video);
  cropped_frame = crop_to_size(arr_frame, 64, 64);
  tensor_frame = tf.tensor(cropped_frame);
  // do stuff with tensor
}

可以通过如下按钮调用

<video id="cam" autoplay muted></video>
<button onclick="predict_img()">
  predict current frame
</button>

Answer 1

是的。你能行的。在

<video>

上播放网络摄像头，然后将图像放在

<canvas>

上，然后获取其数据。在调整大小之间，并删除 Alpha 通道。

const video = document.querySelector('video');
const canvas = document.querySelector('canvas');
const ctx = canvas.getContext('2d');

var constraints = {
  audio: false
};

navigator.mediaDevices.getUserMedia(constraints).then(function(stream) {
  video.srcObject = stream;
}).catch((err) => {
  console.log("playing video instead of webcam")
  video.crossOrigin = "anonymous"
  video.src = "http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4";

});

function snapshot() {

  // resize and crop and center
  const aspectRatio = video.videoWidth / video.videoHeight;
  const targetWidth = 64
  const targetHeight = 64
  let newWidth, newHeight, startX, startY;
  if (aspectRatio > 1) {
    newWidth = targetWidth;
    newHeight = targetHeight / aspectRatio;
    startX = 0;
    startY = (targetHeight - newHeight) / 2; 
  } else {
    newWidth = targetWidth * aspectRatio;
    newHeight = targetHeight;
    startX = (targetWidth - newWidth) / 2; 
    startY = 0;
  }

  // clear canvas 
  ctx.fillStyle = 'black';
  ctx.fillRect(0, 0, canvas.width, canvas.height);

  ctx.drawImage(video, startX, startY, newWidth, newHeight);

  // imageData is 64 x 64 x 4 = (rgba)
  const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height);

  // convert to 64x64x3
  const pixels = [];
  for (let i = 0; i < imageData.data.length; i += 4) {
    pixels.push(imageData.data[i]);
    pixels.push(imageData.data[i + 1]);
    pixels.push(imageData.data[i + 2]);
  }

  console.log("" + pixels)
}

<video autoplay muted style="width: 300px; height: 150px"></video>
<canvas width="64" height="64" style="border: 1px solid red"></canvas>
<button onclick="snapshot()">Take a snapshot</button>