为什么使用 OpenCV 解码视频时帧时间会随着时间增加？

Question

我已经在我的项目中设置了 OpenCV。我将 OpenCV github 存储库添加为项目中的子模块，并将其包含在我的 cmake 依赖项文件中，如下所示：

 set(WITH_FFMPEG ON)
 set(VIDEOIO_PLUGIN_LIST "ffmpeg")
 set(BUILD_PERF_TESTS OFF)
 set(BUILD_TESTS OFF)
 set(INSTALL_TESTS OFF)
 add_subdirectory(${COJE_SRC_DIR}/vendor/opencv build/build_opencv)

然后，我设置了一个继承自 IEntity 的 Video 结构（以使其与我的渲染驱动程序绘制系统一起工作），如下所示：

#pragma once
#include <opencv2/opencv.hpp>
#include <coje/interfaces/IEntity.hpp>
#include <coje/enums/EFileLocation.hpp>
#include <coje/String.hpp>
#include <coje/graphics/Texture.hpp>

namespace coje::entitys
{
 struct Video : IEntity
 {
  String filePath;
  EFileLocation fileLocation;
  String tempname;
  glm::vec2 size;
  UniquePointer<cv::VideoCapture> videoCapturePointer;
  cv::Mat frame;
  cv::Mat frameConverted;
  Floating64 fps = 0;
  Floating64 frameCount = 0;
  Integer64 currentFrameIndex = -1;
  Video(const String &filePath, const EFileLocation &fileLocation, const glm::vec2 &size, const glm::vec3 &position, const glm::quat &rotation);
  ~Video();
  void updateTextureWithFrame(const uInteger64 &frameIndex, UniquePointer<Texture> &texturePointer);
  const Boolean resize(const glm::vec2 &size);
  Boolean update(const uInteger64 &elapsedTimeMs);
 };
}

Video.cpp的来源是：

#include <coje/bullet.hpp>
#include <coje/Common.hpp>
#include <coje/Entitys/Video.hpp>
#include <coje/Logger.hpp>
#include <coje/Timer.hpp>
#include <cstdio>
using namespace coje::entitys;
/*
 */
Video::Video(const String &filePath, const EFileLocation &fileLocation, const glm::vec2 &size, const glm::vec3 &position, const glm::quat &rotation) : IEntity(EntityType)
{
 this->position = position;
 this->rotation = rotation;
 File videoFile(filePath, fileLocation, "r");
 auto videoBytes = videoFile.toBytes();
 tempname = std::tmpnam(0);
 {
  File tempFile(tempname, EFileLocation::Relative, "w");
  tempFile & videoBytes;
 }
 videoCapturePointer = {ReleaseType::Delete, new cv::VideoCapture(tempname.c_str(), cv::CAP_FFMPEG), 1};
 auto &videoCapture = *videoCapturePointer.pointer;
 if (!videoCapture.isOpened())
 {
  Logger(LogType::ERROR, "%s\n", "Error opening video stream from memory");
  return;
 }
 // videoCapture.set(cv::CAP_PROP_BUFFERSIZE, 100);
 uInteger64 bufferSize = videoCapture.get(cv::CAP_PROP_BUFFERSIZE);
 Logger(LogType::INFO, "BufferSize: %llu\n", bufferSize);
 fps = videoCapture.get(cv::CAP_PROP_FPS);
 frameCount = videoCapture.get(cv::CAP_PROP_FRAME_COUNT);
 uInteger64 frameWidth = videoCapture.get(cv::CAP_PROP_FRAME_WIDTH),
            frameHeight = videoCapture.get(cv::CAP_PROP_FRAME_HEIGHT);
 resize(size);
 textures.push_back({ReleaseType::Delete, new Texture(frameWidth, frameHeight, ETextureFormat::RGB8, ETextureType::UnsignedByte), 1});
 glm::ivec3 *indices = (glm::ivec3 *)(*this).operator()<uint32_t>(IEntity::Quanta::Indice, 2);
 indices[0] = {3, 2, 1}; // front
 indices[1] = {1, 0, 3};
 glm::vec2 *uvs = (glm::vec2 *)(*this).operator()<float>(IEntity::Quanta::UV2, 4);
 auto _uvs = Common::getUVs2DQuad();
 for (int index = 0; index < 4; index++)
 {
  uvs[index] = _uvs._data[index];
 }
 TimerFunctions::addFunction({this, &Video::update}, 0, 1000 / fps);
 return;
};
/*
 */
Video::~Video()
{
 File tempFile(tempname);
 tempFile.remove();
};
/*
 */
void Video::updateTextureWithFrame(const uInteger64 &frameIndex, UniquePointer<Texture> &texturePointer)
{
 auto start = std::chrono::high_resolution_clock::now();
 auto &videoCapture = *videoCapturePointer.pointer;
 videoCapture.set(cv::CAP_PROP_POS_FRAMES, frameIndex);
 Boolean frameGrabSuccess = videoCapture.grab();
 if (!frameGrabSuccess)
 {
  Logger(LogType::ERROR, "%s\n", "Failed to grab frame from VideoCapture");
  return;
 }
 Boolean frameRetrieveSuccess = videoCapture.retrieve(frame);
 if (!frameRetrieveSuccess)
 {
  Logger(LogType::ERROR, "%s\n", "Failed to retrieve frame from VideoCapture");
  return;
 }
 auto end = std::chrono::high_resolution_clock::now();
 std::chrono::duration<double, std::milli> elapsed = end - start;
 std::cout << "Video::updateTextureWithFrame took " << elapsed.count() << "ms\n";
 cv::cvtColor(frame, frameConverted, cv::COLOR_BGR2RGB);
 cv::flip(frameConverted, frameConverted, 0);
 if (texturePointer.pointer)
 {
  auto &texture = texturePointer.pointer;
  if (texture->width != frameConverted.cols || texture->height != frameConverted.rows)
  {
   goto _newTexture;
  }
  else
  {
   texture->update(frameConverted.data);
  }
 }
 else
 {
 _newTexture:
  texturePointer = {ReleaseType::Delete, new Texture(frameConverted.cols, frameConverted.rows, frameConverted.data, ETextureFormat::RGB8, ETextureType::UnsignedByte), 1};
 }
};
/*
 */
const Boolean Video::resize(const glm::vec2 &_size)
{
 size = _size;
 glm::vec3 *vertices = (glm::vec3 *)(*this).operator()<float>(IEntity::Quanta::Vertex, 4);
 glm::vec3 topRight = {size.x / 2, size.y / 2, 0};
 glm::vec3 bottomRight = {size.x / 2, -(size.y / 2), 0};
 glm::vec3 bottomLeft = {-(size.x / 2), -(size.y / 2), 0};
 glm::vec3 topLeft = {-(size.x / 2), size.y / 2, 0};
 vertices[0] = topRight;
 vertices[1] = bottomRight;
 vertices[2] = bottomLeft;
 vertices[3] = topLeft;
 *changedPointer = true;
 return true;
};
/*
 */
Boolean Video::update(const uInteger64 &elapsedTimeMs)
{
 currentFrameIndex++;
 Logger(LogType::INFO, "Video-elapsedTime: %llums\n", elapsedTimeMs);
 if (currentFrameIndex == frameCount - 1)
 {
  return false;
 }
 auto &texturePointer = textures._data[0];
 updateTextureWithFrame(currentFrameIndex, texturePointer);
 return true;
};

当运行一个简单的测试视频@

1280x720

时，

updateTextureWithFrame

计时器从12毫秒开始，但随着时间的推移逐渐增加到超过100毫秒甚至更长。导致视频播放速度低于定义的每秒帧数。

是什么导致

updateTextureWithFrame

逐渐增加？我该如何解决？

编辑：

 uInteger64 bufferSize = videoCapture.get(cv::CAP_PROP_BUFFERSIZE);
 Logger(LogType::INFO, "BufferSize: %llu\n", bufferSize);

打印BufferSize: 0。表示ffmpeg不支持设置CAP_PROP_BUFFERSIZE

编辑2： 一些计时日志

Video::updateTextureWithFrame took 16.5161ms
Video::updateTextureWithFrame took 21.6109ms
Video::updateTextureWithFrame took 21.1443ms
Video::updateTextureWithFrame took 20.4253ms
Video::updateTextureWithFrame took 23.9015ms
Video::updateTextureWithFrame took 22.1348ms
Video::updateTextureWithFrame took 21.3723ms
Video::updateTextureWithFrame took 21.2186ms
Video::updateTextureWithFrame took 24.0211ms
Video::updateTextureWithFrame took 24.5907ms
Video::updateTextureWithFrame took 23.2134ms
Video::updateTextureWithFrame took 25.6763ms
Video::updateTextureWithFrame took 25.416ms
Video::updateTextureWithFrame took 25.2314ms
Video::updateTextureWithFrame took 26.3919ms
Video::updateTextureWithFrame took 24.1883ms
Video::updateTextureWithFrame took 27.7095ms
Video::updateTextureWithFrame took 26.5594ms
Video::updateTextureWithFrame took 26.6618ms
Video::updateTextureWithFrame took 29.496ms
Video::updateTextureWithFrame took 27.2731ms
Video::updateTextureWithFrame took 27.5113ms
Video::updateTextureWithFrame took 30.2855ms
Video::updateTextureWithFrame took 27.6773ms
Video::updateTextureWithFrame took 30.5532ms
Video::updateTextureWithFrame took 32.6858ms
Video::updateTextureWithFrame took 32.8735ms
Video::updateTextureWithFrame took 31.7369ms
Video::updateTextureWithFrame took 31.2453ms
Video::updateTextureWithFrame took 30.9424ms
Video::updateTextureWithFrame took 36.7046ms
Video::updateTextureWithFrame took 33.6224ms
Video::updateTextureWithFrame took 32.0368ms
Video::updateTextureWithFrame took 33.0109ms
Video::updateTextureWithFrame took 32.2155ms
Video::updateTextureWithFrame took 33.5314ms
Video::updateTextureWithFrame took 33.576ms
Video::updateTextureWithFrame took 37.8993ms
Video::updateTextureWithFrame took 33.9495ms
Video::updateTextureWithFrame took 35.776ms
Video::updateTextureWithFrame took 36.2566ms
Video::updateTextureWithFrame took 36.5887ms
Video::updateTextureWithFrame took 40.0839ms
Video::updateTextureWithFrame took 38.5146ms
Video::updateTextureWithFrame took 40.72ms
Video::updateTextureWithFrame took 37.8345ms
Video::updateTextureWithFrame took 37.9925ms
Video::updateTextureWithFrame took 39.0402ms
Video::updateTextureWithFrame took 39.8856ms
Video::updateTextureWithFrame took 41.3421ms
Video::updateTextureWithFrame took 41.0703ms
Video::updateTextureWithFrame took 42.9482ms
Video::updateTextureWithFrame took 42.9199ms
Video::updateTextureWithFrame took 44.2593ms
Video::updateTextureWithFrame took 41.2746ms
Video::updateTextureWithFrame took 45.7017ms
Video::updateTextureWithFrame took 46.1854ms
Video::updateTextureWithFrame took 44.154ms
Video::updateTextureWithFrame took 42.6004ms
Video::updateTextureWithFrame took 47.2442ms
Video::updateTextureWithFrame took 43.4156ms
Video::updateTextureWithFrame took 47.9288ms
Video::updateTextureWithFrame took 45.3475ms
Video::updateTextureWithFrame took 46.9646ms
Video::updateTextureWithFrame took 48.4978ms
Video::updateTextureWithFrame took 45.1322ms
Video::updateTextureWithFrame took 48.1365ms
Video::updateTextureWithFrame took 49.8857ms
Video::updateTextureWithFrame took 47.4854ms
Video::updateTextureWithFrame took 48.2378ms
Video::updateTextureWithFrame took 50.9174ms
Video::updateTextureWithFrame took 52.347ms
Video::updateTextureWithFrame took 51.6252ms
Video::updateTextureWithFrame took 52.2018ms
Video::updateTextureWithFrame took 49.2384ms
Video::updateTextureWithFrame took 50.9491ms
Video::updateTextureWithFrame took 52.2139ms
Video::updateTextureWithFrame took 53.3229ms
Video::updateTextureWithFrame took 56.0199ms
Video::updateTextureWithFrame took 55.582ms
Video::updateTextureWithFrame took 55.2675ms
Video::updateTextureWithFrame took 54.9446ms
Video::updateTextureWithFrame took 54.7955ms
Video::updateTextureWithFrame took 54.0296ms
Video::updateTextureWithFrame took 54.0375ms
Video::updateTextureWithFrame took 57.0916ms
Video::updateTextureWithFrame took 55.2474ms
Video::updateTextureWithFrame took 56.8046ms
Video::updateTextureWithFrame took 57.562ms
Video::updateTextureWithFrame took 59.9115ms
Video::updateTextureWithFrame took 59.3991ms
Video::updateTextureWithFrame took 60.0536ms
Video::updateTextureWithFrame took 59.9457ms
Video::updateTextureWithFrame took 57.5088ms
Video::updateTextureWithFrame took 59.1255ms
Video::updateTextureWithFrame took 62.2311ms
Video::updateTextureWithFrame took 59.0422ms
Video::updateTextureWithFrame took 62.0419ms
Video::updateTextureWithFrame took 62.0586ms
Video::updateTextureWithFrame took 64.0988ms
Video::updateTextureWithFrame took 64.743ms
Video::updateTextureWithFrame took 63.008ms
Video::updateTextureWithFrame took 65.1726ms
Video::updateTextureWithFrame took 63.3618ms
Video::updateTextureWithFrame took 65.6431ms
Video::updateTextureWithFrame took 63.8957ms
Video::updateTextureWithFrame took 65.1142ms
Video::updateTextureWithFrame took 67.2243ms
Video::updateTextureWithFrame took 65.1302ms
Video::updateTextureWithFrame took 66.4947ms
Video::updateTextureWithFrame took 66.092ms
Video::updateTextureWithFrame took 68.6997ms
Video::updateTextureWithFrame took 70.5683ms
Video::updateTextureWithFrame took 71.9019ms
Video::updateTextureWithFrame took 68.6088ms
Video::updateTextureWithFrame took 70.7946ms
Video::updateTextureWithFrame took 68.263ms
Video::updateTextureWithFrame took 66.1565ms
Video::updateTextureWithFrame took 70.6742ms
Video::updateTextureWithFrame took 70.7035ms
Video::updateTextureWithFrame took 73.8002ms
Video::updateTextureWithFrame took 73.1897ms
Video::updateTextureWithFrame took 74.006ms
Video::updateTextureWithFrame took 74.1048ms
Video::updateTextureWithFrame took 72.9378ms
Video::updateTextureWithFrame took 75.0651ms
Video::updateTextureWithFrame took 73.5676ms
Video::updateTextureWithFrame took 73.7706ms
Video::updateTextureWithFrame took 74.0839ms
Video::updateTextureWithFrame took 74.6773ms
Video::updateTextureWithFrame took 75.8827ms
Video::updateTextureWithFrame took 74.4724ms
Video::updateTextureWithFrame took 75.2119ms
Video::updateTextureWithFrame took 83.4102ms
Video::updateTextureWithFrame took 77.6811ms
Video::updateTextureWithFrame took 78.7307ms
Video::updateTextureWithFrame took 80.1705ms
Video::updateTextureWithFrame took 78.6064ms
Video::updateTextureWithFrame took 80.803ms
Video::updateTextureWithFrame took 80.0117ms
Video::updateTextureWithFrame took 78.2948ms
Video::updateTextureWithFrame took 81.0375ms
Video::updateTextureWithFrame took 78.7389ms
Video::updateTextureWithFrame took 80.2201ms
Video::updateTextureWithFrame took 82.8578ms
Video::updateTextureWithFrame took 84.2388ms
Video::updateTextureWithFrame took 84.6484ms
Video::updateTextureWithFrame took 87.6683ms
Video::updateTextureWithFrame took 82.8939ms
Video::updateTextureWithFrame took 84.015ms
Video::updateTextureWithFrame took 88.1832ms
Video::updateTextureWithFrame took 83.3894ms
Video::updateTextureWithFrame took 86.9088ms
Video::updateTextureWithFrame took 87.1049ms
Video::updateTextureWithFrame took 87.6748ms
Video::updateTextureWithFrame took 87.178ms
Video::updateTextureWithFrame took 84.7988ms
Video::updateTextureWithFrame took 89.528ms
Video::updateTextureWithFrame took 88.7021ms
Video::updateTextureWithFrame took 90.0357ms
Video::updateTextureWithFrame took 90.398ms
Video::updateTextureWithFrame took 87.8047ms
Video::updateTextureWithFrame took 90.2447ms
Video::updateTextureWithFrame took 94.6288ms
Video::updateTextureWithFrame took 88.9265ms
Video::updateTextureWithFrame took 89.01ms
Video::updateTextureWithFrame took 87.6294ms
Video::updateTextureWithFrame took 90.6988ms
Video::updateTextureWithFrame took 93.0173ms
Video::updateTextureWithFrame took 92.1651ms
Video::updateTextureWithFrame took 92.9234ms
Video::updateTextureWithFrame took 95.4223ms
Video::updateTextureWithFrame took 99.0941ms
Video::updateTextureWithFrame took 97.3014ms
Video::updateTextureWithFrame took 91.8709ms
Video::updateTextureWithFrame took 96.8951ms
Video::updateTextureWithFrame took 95.3506ms
Video::updateTextureWithFrame took 96.5474ms
Video::updateTextureWithFrame took 92.4739ms
Video::updateTextureWithFrame took 95.1857ms
Video::updateTextureWithFrame took 96.6743ms
Video::updateTextureWithFrame took 99.0657ms
Video::updateTextureWithFrame took 105.84ms
Video::updateTextureWithFrame took 99.3163ms
Video::updateTextureWithFrame took 127.942ms
Video::updateTextureWithFrame took 101.378ms
Video::updateTextureWithFrame took 98.6114ms
Video::updateTextureWithFrame took 101.161ms
Video::updateTextureWithFrame took 102.271ms
Video::updateTextureWithFrame took 100.77ms
Video::updateTextureWithFrame took 100.825ms
Video::updateTextureWithFrame took 100.64ms
Video::updateTextureWithFrame took 99.7002ms
Video::updateTextureWithFrame took 103.207ms
Video::updateTextureWithFrame took 107.135ms
Video::updateTextureWithFrame took 100.766ms
Video::updateTextureWithFrame took 103.321ms
Video::updateTextureWithFrame took 107.361ms
Video::updateTextureWithFrame took 104.086ms
Video::updateTextureWithFrame took 100.975ms
Video::updateTextureWithFrame took 105.846ms
Video::updateTextureWithFrame took 104.755ms
Video::updateTextureWithFrame took 105.893ms
Video::updateTextureWithFrame took 105.234ms
Video::updateTextureWithFrame took 109.415ms
Video::updateTextureWithFrame took 107.942ms
Video::updateTextureWithFrame took 109.816ms
Video::updateTextureWithFrame took 109.268ms
Video::updateTextureWithFrame took 111.918ms
Video::updateTextureWithFrame took 110.123ms
Video::updateTextureWithFrame took 109.975ms
Video::updateTextureWithFrame took 110.105ms
Video::updateTextureWithFrame took 115.888ms
Video::updateTextureWithFrame took 112.443ms
Video::updateTextureWithFrame took 111.795ms
Video::updateTextureWithFrame took 112.016ms
Video::updateTextureWithFrame took 115.857ms
Video::updateTextureWithFrame took 114.762ms
Video::updateTextureWithFrame took 112.551ms
Video::updateTextureWithFrame took 116.05ms
Video::updateTextureWithFrame took 119.133ms
Video::updateTextureWithFrame took 114.202ms
Video::updateTextureWithFrame took 119.864ms
Video::updateTextureWithFrame took 119.743ms
Video::updateTextureWithFrame took 119.911ms
Video::updateTextureWithFrame took 120.957ms
Video::updateTextureWithFrame took 117.611ms
Video::updateTextureWithFrame took 116.596ms
Video::updateTextureWithFrame took 116.859ms
Video::updateTextureWithFrame took 120.355ms
Video::updateTextureWithFrame took 121.932ms
Video::updateTextureWithFrame took 117.56ms
Video::updateTextureWithFrame took 122.747ms
Video::updateTextureWithFrame took 120.103ms
Video::updateTextureWithFrame took 123.497ms
Video::updateTextureWithFrame took 126.391ms
Video::updateTextureWithFrame took 123.512ms
Video::updateTextureWithFrame took 121.612ms
Video::updateTextureWithFrame took 130.169ms
Video::updateTextureWithFrame took 126.936ms
Video::updateTextureWithFrame took 122.812ms
Video::updateTextureWithFrame took 122.843ms
Video::updateTextureWithFrame took 124.214ms
Video::updateTextureWithFrame took 125.563ms
Video::updateTextureWithFrame took 128.024ms
Video::updateTextureWithFrame took 129.263ms
Video::updateTextureWithFrame took 130.028ms
Video::updateTextureWithFrame took 127.493ms
Video::updateTextureWithFrame took 129.553ms
Video::updateTextureWithFrame took 130.538ms
Video::updateTextureWithFrame took 22.6048ms
Video::updateTextureWithFrame took 20.2454ms
Video::updateTextureWithFrame took 19.9947ms
Video::updateTextureWithFrame took 21.2817ms
Video::updateTextureWithFrame took 22.6694ms
Video::updateTextureWithFrame took 25.5187ms
Video::updateTextureWithFrame took 19.8971ms
Video::updateTextureWithFrame took 22.2975ms
Video::updateTextureWithFrame took 21.4979ms
Video::updateTextureWithFrame took 25.6767ms
Video::updateTextureWithFrame took 23.4276ms
Video::updateTextureWithFrame took 25.5657ms
Video::updateTextureWithFrame took 23.2816ms
Video::updateTextureWithFrame took 26.8515ms
Video::updateTextureWithFrame took 24.0271ms
Video::updateTextureWithFrame took 24.4675ms
Video::updateTextureWithFrame took 25.6897ms
Video::updateTextureWithFrame took 28.7489ms
Video::updateTextureWithFrame took 24.6164ms
Video::updateTextureWithFrame took 29.6739ms
Video::updateTextureWithFrame took 27.8118ms
Video::updateTextureWithFrame took 30.3992ms
Video::updateTextureWithFrame took 28.2943ms
Video::updateTextureWithFrame took 29.9693ms
Video::updateTextureWithFrame took 30.6129ms

Answer 1

我完整地向 ChatGPT 发送了这个问题，它给出了这样的回复：

帧查找开销：连续设置帧位置（cv::CAP_PROP_POS_FRAMES）可能非常低效，特别是对于某些编解码器或容器格式。它可能需要解码许多帧才能达到所需的帧。

解决方案：不要每次都设置帧索引，而是在可能的情况下尝试按顺序读取帧，或者考虑将视频重新构造为允许更快查找的格式（如果需要随机访问）。

所以删除线后

videoCapture.set(cv::CAP_PROP_POS_FRAMES, frameIndex);

就成功了！

updateTextureWithFrame

现在以亚毫秒计时运行。