GStreamer 时间戳 (PTS) 对于捕获的帧不是单调递增的

Question

我编写了一些代码来使用 GStreamer 1.0 (PyGST) 从网络摄像头捕获帧。对我来说，知道确切的捕获时间很重要。为此，我设置了 v4l2src 属性 do-timestamp，并使用 appsink 将缓冲区 PTS 写入文本文件。

但是，时间戳并不是单调递增的。例如。第 16 帧的时间戳是 0.88199 秒，第 17 帧的时间戳是 0.77462 秒，即比前一帧早 0.10737 秒。 （我有一张显示问题的图，但缺乏发布它所需的声誉。）

捕获的 GstBuffer 的 PTS 并不总是单调递增，这是否正确？如果这不是正常行为，有人知道我搞砸了什么吗？

我使用 Logitech c920 网络摄像头。这些帧在相机上采用 h.264 编码。代码大致如下：

import gi
gi.require_version('Gst', '1.0')
from gi.repository import GObject, Gst, Gtk
GObject.threads_init()
Gst.init(None)

class Webcam:
def __init__(self, video_dev='/dev/video0', fps=30):

    ts_log_fname = 'webcam_timestamps.log'
    vid_fname = 'webcam.mkv'
    self.ts_log = open(ts_log_fname, 'w')
    self.ts_log.write('video filename: %s\n '
                      '\nframe_number, cam_running_ts\n' % vid_fname)

    self.n_frames = 0

    # Create GStreamer pipline
    self.pipeline = Gst.Pipeline()

    # Create bus to get events from GStreamer pipeline
    self.bus = self.pipeline.get_bus()
    self.bus.add_signal_watch()
    self.bus.connect('message::error', self.on_error)
    self.bus.enable_sync_message_emission()
    self.bus.connect('sync-message::element', self.on_sync_message)

    ###########################
    # Callable function
    ###########################
    def on_new_sample(appsink):
        """
        Function called from the pipeline by appsink.
        Writes the timestampes of frame capture to a log file.
        """
        # Get the buffer
        smp = appsink.emit('pull-sample')
        buf = smp.get_buffer()
        # Nanoseconds to seconds
        timestamp = np.float64(1e-9) * buf.pts
        self.n_frames += 1
        self.ts_log.write('%d,%0.9f\n' % (self.n_frames, timestamp))
        return False

    ###########################
    # Create GStreamer elements
    ###########################
    # Video source:
    self.v4l2src0 = Gst.ElementFactory.make('v4l2src', None)
    self.v4l2src0.set_property('device', video_dev)
    self.v4l2src0.set_property('do-timestamp', 'true')
    # Video source filters:
    vid0caps = Gst.Caps.from_string('video/x-h264,width=%d,height=%d,'
                                    'framerate=%d/1' % (1280, 720, fps))
    self.vid0filter = Gst.ElementFactory.make('capsfilter', None)
    self.vid0filter.set_property('caps', vid0caps)
    # Parse video:
    self.vid0parse = Gst.ElementFactory.make('h264parse', None)
    # Split:
    self.tee0 = Gst.ElementFactory.make('tee', None)
    self.tee0.set_property('name', 't0')
    ####
    # Display branch
    ####
    # Decode
    self.vid0decode = Gst.ElementFactory.make('avdec_h264', None)
    # Scale to display size:
    self.disp0scale = Gst.ElementFactory.make('videoscale', None)
    # Display filter caps:
    disp0caps = Gst.Caps.from_string('video/x-raw,width=%d,height=%d' %
                                     (800, 600))
    # Sinks:
    self.disp0sink = Gst.ElementFactory.make('autovideosink', None)
    self.disp0sink.set_property('filter-caps', disp0caps)
    ####
    # File branch
    ####
    self.mux = Gst.ElementFactory.make('matroskamux', None)
    self.file0sink = Gst.ElementFactory.make('filesink', None)
    self.file0sink.set_property('location', vid_fname)
    self.file0sink.set_property('sync', False)
    ####
    # Timestamp branch
    ####
    # Create appsink
    self.ts0sink = Gst.ElementFactory.make('appsink', None)
    # Setting properties of appsink
    ts0caps = vid0caps  # use same caps as for camera
    self.ts0sink.set_property('caps', ts0caps)
    self.ts0sink.set_property("max-buffers", 20)  # Limit memory usage
    # Tell sink to emit signals
    self.ts0sink.set_property('emit-signals', True)
    self.ts0sink.set_property('sync', False)  # No sync
    # Connect appsink to my function (writing timestamps)
    self.ts0sink.connect('new-sample', on_new_sample)

    self.queue0 = Gst.ElementFactory.make('queue', None)
    self.queue1 = Gst.ElementFactory.make('queue', None)                
    self.disp_queue = Gst.ElementFactory.make('queue', None)
    self.file_queue = Gst.ElementFactory.make('queue', None)
    self.ts_queue = Gst.ElementFactory.make('queue', None)

    # Add elements to the pipeline
    self.pipeline.add(self.v4l2src0)
    self.pipeline.add(self.vid0filter)
    self.pipeline.add(self.vid0parse)
    self.pipeline.add(self.tee0)
    self.pipeline.add(self.vid0decode)
    self.pipeline.add(self.disp0scale)
    self.pipeline.add(self.disp0sink)
    self.pipeline.add(self.mux)
    self.pipeline.add(self.file0sink)
    self.pipeline.add(self.ts0sink)
    self.pipeline.add(self.queue0)
    self.pipeline.add(self.queue1)
    self.pipeline.add(self.disp_queue)
    self.pipeline.add(self.file_queue)
    self.pipeline.add(self.ts_queue)

    ###############
    # Link elements
    ###############
    # video source
    if not self.v4l2src0.link(self.vid0filter):
        print('video source to video filter link failed')          
    if not self.vid0filter.link(self.vid0parse):
        print('video filter to video parse link failed')
    if not self.vid0parse.link(self.tee0):
        print('video parse to tee link failed')    
    # tee
    if not self.tee0.link(self.disp_queue):
        print('tee to display queue link failed')
    if not self.tee0.link(self.file_queue):
        print('tee to file queue link failed')
    if not self.tee0.link(self.ts_queue):
        print('tee to ts queue link failed')
    # video display sink
    if not self.disp_queue.link(self.vid0decode):
        print('dispaly queue to video decode link failed')
    if not self.vid0decode.link(self.disp0scale):
        print('decode to videoscale link failed')
    if not self.disp0scale.link(self.queue0):
        print('disp0scale to queue0 link failed')            
    if not self.queue0.link_filtered(self.disp0sink, disp0caps):
        print('queue0 to display-sink link failed')
    # file sink
    if not self.file_queue.link(self.mux):
        print('file queue to mux link failed')           
    if not self.mux.link(self.queue1):
        print('mux to queue1 link failed')            
    if not self.queue1.link(self.file0sink):
        print('queue1 to file-sink link failed')
    # timestamp sink
    if not self.ts_queue.link(self.ts0sink):
        print('ts queue to ts-sink link failed')

def run(self):
    self.offset_t = datetime.now().timestamp() - self.t_start
    self.pipeline.set_state(Gst.State.PLAYING)

def quit(self):
    self.pipeline.set_state(Gst.State.NULL)
    self.ts_log.close()

def on_sync_message(self, bus, msg):
    if msg.get_structure().get_name() == 'prepare-window-handle':
        msg.src.set_property('force-aspect-ratio', True)

def on_error(self, bus, msg):
    print('on_error():', msg.parse_error())

Answer 1

如果有人试图修复来自 RTSP 摄像机 (30 FPS) 的传入流的 PTS 时间戳，这里的代码将允许您动态更改 PTS。我花了两天时间才弄清楚这一点，希望它能为其他人节省一些时间。

2023 年 8 月更新：现在有一个更优雅的解决方案（https://stackoverflow.com/a/76861526/1708124）

#!/usr/bin/python3

import sys
import logging
import numpy as np

import gi
gi.require_version("Gst", "1.0")
gi.require_version("GstApp", "1.0")
from gi.repository import Gst, GstApp, GObject, GLib

Gst.init(None)

last_pts = 0

class Main:
    def __init__(self):

        self.signed = None
        self.depth = None
        self.rate = None
        self.channels = None

        self.pipeline = Gst.parse_launch("rtspsrc latency=2000 location=rtspt://guest:[email protected]:5540/Streaming/Channels/101 ! rtph264depay ! h264parse ! avdec_h264 ! identity name=adjust ! queue2 ! watchdog timeout=10000 ! autovideosink")

        src_element = self.pipeline.get_by_name('adjust')
        #get the static source pad of the element
        srcpad = src_element.get_static_pad('src')
        #add the probe to the pad obtained in previous solution
        probeID = srcpad.add_probe(Gst.PadProbeType.BUFFER, self.adjust_pts)
            
        # The MainLoop
        self.mainloop = GLib.MainLoop()

        # And off we go!
        self.pipeline.set_state(Gst.State.PLAYING)
        self.mainloop.run()


    def adjust_pts(self, pad, info):
        global last_pts

        # 33333333 nanoseconds is 0.03 seconds per frame (at 30 FPS)
        new_pts = last_pts + 33333333

        info.get_buffer().pts = new_pts

        last_pts = new_pts

        return Gst.PadProbeReturn.OK



    def on_eos(self, bus, msg):
        logging.debug('on_eos')
        self.mainloop.quit()

    def on_error(self, bus, msg):
        error = msg.parse_error()
        print('on_error:', error[1])
        self.mainloop.quit()



class Error(Exception):

    def __init__(self, message, detail=None):
        global last_detail

        self.message = message
        if detail:
            self.detail = detail
        else:
            self.detail = last_detail

        logging.debug('audio: Error %s %s', self.message, self.detail)

    def __str__(self):
        return '%s - %s' % (self.message, self.detail)


Main()

Answer 2

根据 mpr 的建议，流式传输 h.264 时会出现非单调时间戳。它似乎也发生在流的开头。当流式传输原始视频时，问题就消失了。

每 30 帧左右就会出现 300 毫秒的跳动，这似乎是 Logitech C920 摄像头特有的现象。当我使用板载笔记本电脑摄像头时，跳跃更小，为 130 毫秒，甚至更少，每 85 帧左右一次。

Answer 3

我知道这有点晚了 kalleknast，你有没有设法获得时间戳？我用

do-timestamp=True

代替

rtspsrc

但没用。

Answer 4

我花了两年时间从不同的方向解决这个问题。我最终解决了这个问题：

1. 测量 RTSP 流的实际 FPS（我在此处发布了代码 - https://github.com/ethaniel/rtsp-fps）。 （运行此脚本一分钟左右，直到

   fps

   值稳定）
2. 使用

   videorate

   根据上面脚本提供的值设置帧速率：

gst-launch-1.0 rtspsrc location=rtspt://admin:[email protected]/Streaming/Channels/101 ! \ 
rtph264depay ! h264parse ! avdec_h264 ! \
videorate ! video/x-raw,framerate=30039/1000 ! \
watchdog timeout=10000 ! autovideosink

所以这整件事的作用是，即使 PTS 在某些帧之间不是单调增加，gstreamer 也可以根据您计算的平均值来修复它。