utils/frame_queue: add PTS drift compensation

By default, allows for correcting drift/jitter of up to 1 ms per frame,
in line with expected jitter from e.g. Matroska sources. This is enough
to cover the mismatch between 23.976 Hz and 24.000 Hz, but not the
mismatch between 24 and 25 Hz (which is just over 1ms per frame).

There are two use cases in which this code will be useful:

1. Displaying 23.976 Hz video files on e.g. 60.000 Hz monitor (or any
   other near-miss), in display-timed mode. Given the usual 3:2
   cadence, after five vsyncs (three source frames), we will have:

   Vsync (wallclock) PTS = 83.33333333333334
   Frame PTS = 83.41675008341676

   This is a difference of 0.1ms, and will therefore be fixed by this
   code, adjusting all future wallclock PTS up by the cumulative drift
   (0.1ms).

2. Displaying any video with some amount of jitter on the PTS
   measurements. This avoids some situation where we have frames with
   weights 0.999 and 0.001 for example. Normally this should get rounded
   away by the frame mixing code anyway, but it's better to have an
   exact match.

demos/plplay.c

@@ -468,10 +468,10 @@ static bool render_loop(struct plplay *p)
 {
     pl_options opts = p->opts;
 
-    struct pl_queue_params qparams = {
+    struct pl_queue_params qparams = *pl_queue_params(
         .interpolation_threshold = 0.01,
         .timeout = UINT64_MAX,
-    };
+    );
 
     // Initialize the frame queue, blocking indefinitely until done
     struct pl_frame_mix mix;

demos/settings.c

@@ -1158,6 +1158,8 @@ void update_settings(struct plplay *p, const struct pl_frame *target)
                 nk_labelf(nk, NK_TEXT_LEFT, "%.3f", pl_queue_estimate_fps(p->queue));
                 nk_label(nk, "Estimated vsync rate:", NK_TEXT_LEFT);
                 nk_labelf(nk, NK_TEXT_LEFT, "%.3f", pl_queue_estimate_vps(p->queue));
+                nk_label(nk, "PTS drift offset:", NK_TEXT_LEFT);
+                nk_labelf(nk, NK_TEXT_LEFT, "%.3f ms", 1e3 * pl_queue_pts_offset(p->queue));
                 nk_label(nk, "Frames rendered:", NK_TEXT_LEFT);
                 nk_labelf(nk, NK_TEXT_LEFT, "%"PRIu32, p->stats.rendered);
                 nk_label(nk, "Decoded frames", NK_TEXT_LEFT);

meson.build

@@ -12,6 +12,7 @@ project('libplacebo', ['c', 'cpp'],
     7,
     # API version
     {
+      '340': 'add pl_queue_params.drift_compensation, PL_QUEUE_DEFAULTS and pl_queue_pts_offset',
       '339': 'add pl_peak_detect_params.black_cutoff',
       '338': 'split pl_filter_nearest into pl_filter_nearest and pl_filter_box',
       '337': 'fix PL_FILTER_DOWNSCALING constant',

src/include/libplacebo/utils/frame_queue.h

@@ -150,6 +150,12 @@ struct pl_queue_params {
     // between calls to `pl_queue_update`. (Optional)
     float vsync_duration;
 
+    // If the difference between `pts` and the closest frame is smaller than
+    // this delta (in seconds), the mismatch will be assumed as drift/jitter
+    // and dynamically subtracted from all future pl_queue_update calls, until
+    // the queue is either reset or the PTS jumps by a large amount. (Optional)
+    float drift_compensation;
+
     // If the difference between the (estimated) vsync duration and the
     // (measured) frame duration is smaller than this threshold, silently
     // disable interpolation and switch to ZOH semantics instead.
@@ -181,7 +187,10 @@ struct pl_queue_params {
     void *priv;
 };
 
-#define pl_queue_params(...) (&(struct pl_queue_params) { __VA_ARGS__ })
+#define PL_QUEUE_DEFAULTS       \
+    .drift_compensation = 1e-3,
+
+#define pl_queue_params(...) (&(struct pl_queue_params) { PL_QUEUE_DEFAULTS __VA_ARGS__ })
 
 // Advance the frame queue's internal state to the target timestamp. Any frames
 // which are no longer needed (i.e. too far in the past) are automatically
@@ -214,6 +223,11 @@ PL_API float pl_queue_estimate_vps(pl_queue queue);
 // Returns the number of frames currently contained in a pl_queue.
 PL_API int pl_queue_num_frames(pl_queue queue);
 
+// Returns the current PTS offset factor, as determined by the PTS drift
+// compensation algorithm. This value is added onto all incoming values of
+// pl_queue_params.pts.
+PL_API double pl_queue_pts_offset(pl_queue queue);
+
 // Inspect the contents of the Nth queued frame. Returns false if `idx` is
 // out of range.
 //

src/utils/frame_queue.c

@@ -97,6 +97,7 @@ struct pl_queue_t {
     float reported_vps;
     float reported_fps;
     double prev_pts;
+    double pts_offset;
 
     // Storage for temporary arrays
     PL_ARRAY(uint64_t) tmp_sig;
@@ -929,6 +930,7 @@ static bool prefill(pl_queue p, const struct pl_queue_params *params)
 enum pl_queue_status pl_queue_update(pl_queue p, struct pl_frame_mix *out_mix,
                                      const struct pl_queue_params *params)
 {
+    struct pl_queue_params fixed;
     pl_mutex_lock(&p->lock_strong);
     pl_mutex_lock(&p->lock_weak);
     default_estimate(&p->vps, params->vsync_duration);
@@ -957,6 +959,7 @@ enum pl_queue_status pl_queue_update(pl_queue p, struct pl_frame_mix *out_mix,
         // the FPS estimate, treat this as a new frame.
         PL_TRACE(p, "Discontinuous target PTS jump %f -> %f, ignoring...",
                  p->prev_pts, params->pts);
+        p->pts_offset = 0.0;
 
     } else if (delta > 0) {
 
@@ -966,6 +969,22 @@ enum pl_queue_status pl_queue_update(pl_queue p, struct pl_frame_mix *out_mix,
 
     p->prev_pts = params->pts;
 
+    if (params->drift_compensation > 0.0f) {
+        // Adjust PTS offset if PTS is near-match for existing frame
+        double pts = params->pts + p->pts_offset;
+        for (int i = 0; i < p->queue.num; i++) {
+            if (fabs(p->queue.elem[i]->pts - pts) < params->drift_compensation) {
+                p->pts_offset = p->queue.elem[i]->pts - params->pts;
+                pts = p->queue.elem[i]->pts;
+                break;
+            }
+        }
+
+        fixed = *params;
+        fixed.pts = pts;
+        params = &fixed;
+    }
+
     // As a special case, prefill the queue if this is the first frame
     if (!params->pts && !p->queue.num) {
         if (!prefill(p, params)) {
@@ -1019,6 +1038,14 @@ int pl_queue_num_frames(pl_queue p)
     return count;
 }
 
+double pl_queue_pts_offset(pl_queue p)
+{
+    pl_mutex_lock(&p->lock_weak);
+    double offset = p->pts_offset;
+    pl_mutex_unlock(&p->lock_weak);
+    return offset;
+}
+
 bool pl_queue_peek(pl_queue p, int idx, struct pl_source_frame *out)
 {
     pl_mutex_lock(&p->lock_weak);