fftools/ffmpeg_sched: track upstream timestamp offsets for pacing

With output-side -ss, audio/video filtering and subtitle encoding
subtract the output start time before packets reach the scheduler.
Streamcopy performs its subtraction later, in the muxer. Comparing
these different timelines can starve copied audio long enough for early
HLS segments to be written without audio packets.

Register the upstream timestamp offset for each encoded mux stream and
add it back in a separate last_dts_paced field. Use that field for both
the trailing timestamp and choke comparisons, while retaining the raw
last_dts for progress reporting. Register subtitle offsets as well,
matching the packet timestamp rebasing introduced by 7f2b4adeac.

Add a generated-input HLS FATE test using native encoders. Declare the
source, transcode and probe dependencies and register the test with
FATE_FFMPEG_FFPROBE. Use a single stream listing in the reference, as
required after 88f7fd9ec5 made program-stream output explicit.

The scheduler regression was introduced by d119ae2fd8.
Fixes #22765.

Assisted-by: Fairy
Signed-off-by: Piotr Niełacny <piotr.nielacny@gmail.com>
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

fftools/ffmpeg_mux_init.c

@@ -1635,6 +1635,12 @@ static int ost_add(Muxer *mux, const OptionsContext *o, enum AVMediaType type,
         }
     }
 
+    // Audio/video filters and subtitle encoding subtract the output start
+    // time before scheduling; streamcopy does so later in the muxer.
+    if (ost->enc && mux->of.start_time != AV_NOPTS_VALUE && ms->sch_idx >= 0)
+        sch_mux_stream_set_dts_pacing_offset(mux->sch, mux->sch_idx,
+                                              ms->sch_idx, mux->of.start_time);
+
     if (src.type != SCH_NODE_TYPE_NONE) {
         ret = sch_connect(mux->sch,
                           src, SCH_MSTREAM(mux->sch_idx, ms->sch_idx));

fftools/ffmpeg_sched.c

@@ -198,12 +198,22 @@ typedef struct SchMuxStream {
     // an EOF was generated while flushing the pre-mux queue
     int                 init_eof;
 
+    // offset (in AV_TIME_BASE_Q) subtracted by the upstream pipeline
+    // (e.g. filter ts_offset), added back when computing last_dts_paced
+    // so that cross-stream pacing compares on a common source timeline
+    int64_t             dts_pacing_offset;
+
     ////////////////////////////////////////////////////////////
     // The following are protected by Scheduler.schedule_lock //
 
     /* dts+duration of the last packet sent to this stream
-       in AV_TIME_BASE_Q */
+       in AV_TIME_BASE_Q, as observed by the muxer (no offset applied);
+       feeds progressing_dts() -> sch->last_dts -> CLI progress display */
     int64_t             last_dts;
+    /* last_dts + dts_pacing_offset, kept on a common source timeline
+       across streams; consumed by trailing_dts() and the choke check
+       in schedule_update_locked() */
+    int64_t             last_dts_paced;
     // this stream no longer accepts input
     int                 source_finished;
     ////////////////////////////////////////////////////////////
@@ -447,10 +457,10 @@ static int64_t trailing_dts(const Scheduler *sch)
 
             if (ms->source_finished)
                 continue;
-            if (ms->last_dts == AV_NOPTS_VALUE)
+            if (ms->last_dts_paced == AV_NOPTS_VALUE)
                 return AV_NOPTS_VALUE;
 
-            min_dts = FFMIN(min_dts, ms->last_dts);
+            min_dts = FFMIN(min_dts, ms->last_dts_paced);
         }
     }
 
@@ -704,7 +714,8 @@ int sch_add_mux_stream(Scheduler *sch, unsigned mux_idx)
     if (!ms->pre_mux_queue.fifo)
         return AVERROR(ENOMEM);
 
-    ms->last_dts = AV_NOPTS_VALUE;
+    ms->last_dts       = AV_NOPTS_VALUE;
+    ms->last_dts_paced = AV_NOPTS_VALUE;
 
     return stream_idx;
 }
@@ -1248,6 +1259,21 @@ void sch_mux_stream_buffering(Scheduler *sch, unsigned mux_idx, unsigned stream_
     ms->pre_mux_queue.data_threshold = data_threshold;
 }
 
+void sch_mux_stream_set_dts_pacing_offset(Scheduler *sch, unsigned mux_idx,
+                                          unsigned stream_idx, int64_t offset)
+{
+    SchMux       *mux;
+    SchMuxStream *ms;
+
+    av_assert0(mux_idx < sch->nb_mux);
+    mux = &sch->mux[mux_idx];
+
+    av_assert0(stream_idx < mux->nb_streams);
+    ms = &mux->streams[stream_idx];
+
+    ms->dts_pacing_offset = offset;
+}
+
 int sch_mux_stream_ready(Scheduler *sch, unsigned mux_idx, unsigned stream_idx)
 {
     SchMux *mux;
@@ -1457,9 +1483,9 @@ static void schedule_update_locked(Scheduler *sch)
             // and not too far ahead of the trailing stream
             if (ms->source_finished)
                 continue;
-            if (dts == AV_NOPTS_VALUE && ms->last_dts != AV_NOPTS_VALUE)
+            if (dts == AV_NOPTS_VALUE && ms->last_dts_paced != AV_NOPTS_VALUE)
                 continue;
-            if (dts != AV_NOPTS_VALUE && ms->last_dts - dts >= SCHEDULE_TOLERANCE)
+            if (dts != AV_NOPTS_VALUE && ms->last_dts_paced - dts >= SCHEDULE_TOLERANCE)
                 continue;
 
             // resolve the source to unchoke
@@ -2053,8 +2079,12 @@ update_schedule:
     if (dts != AV_NOPTS_VALUE || !pkt) {
         pthread_mutex_lock(&sch->schedule_lock);
 
-        if (pkt) ms->last_dts = dts;
-        else     ms->source_finished = 1;
+        if (pkt) {
+            ms->last_dts       = dts;
+            ms->last_dts_paced = dts + ms->dts_pacing_offset;
+        } else {
+            ms->source_finished = 1;
+        }
 
         schedule_update_locked(sch);
 

fftools/ffmpeg_sched.h

@@ -285,6 +285,14 @@ int sch_add_mux_stream(Scheduler *sch, unsigned mux_idx);
 void sch_mux_stream_buffering(Scheduler *sch, unsigned mux_idx, unsigned stream_idx,
                               size_t data_threshold, int max_packets);
 
+/**
+ * Set the timestamp offset (in AV_TIME_BASE_Q) that the upstream pipeline
+ * subtracted from this stream's packets. The scheduler adds it back when
+ * comparing DTS across streams for pacing. Defaults to 0 (streamcopy).
+ */
+void sch_mux_stream_set_dts_pacing_offset(Scheduler *sch, unsigned mux_idx,
+                                          unsigned stream_idx, int64_t offset);
+
 /**
  * Signal to the scheduler that the specified muxed stream is initialized and
  * ready. Muxing is started once all the streams are ready.

tests/fate/hlsenc.mak

@@ -265,7 +265,58 @@ fate-hls-fmp4-big-init: tests/data/hls_fmp4_big_init.m3u8
 fate-hls-fmp4-big-init: CLEANFILES = tests/data/hls_fmp4_big_init.m3u8 tests/data/hls_fmp4_big_init.mp4 tests/data/hls_fmp4_big_init_*.m4s
 fate-hls-fmp4-big-init: CMD = framecrc -i $(TARGET_PATH)/tests/data/hls_fmp4_big_init.m3u8 -map 0:a:99 -c copy
 
+# Regression test for cross-stream pacing with output -ss. The transcoded
+# video's timestamps are rebased before the scheduler observes them, whereas
+# copied audio is rebased by of_streamcopy() after the scheduler observes it.
+# Comparing these different timelines can starve audio in early HLS segments.
+tests/data/hls_sched_v.mkv: TAG = GEN
+tests/data/hls_sched_v.mkv: ffmpeg$(PROGSSUF)$(EXESUF) | tests/data
+	$(M)$(TARGET_EXEC) $(TARGET_PATH)/$< -nostdin \
+	-f lavfi -i "testsrc2=size=128x72:rate=24:d=180" \
+	-c:v mpeg4 -g 500 -flags +bitexact \
+	-an -y $(TARGET_PATH)/tests/data/hls_sched_v.mkv 2>/dev/null
+
+tests/data/hls_sched_a.mka: TAG = GEN
+tests/data/hls_sched_a.mka: ffmpeg$(PROGSSUF)$(EXESUF) | tests/data
+	$(M)$(TARGET_EXEC) $(TARGET_PATH)/$< -nostdin \
+	-f lavfi -i "aevalsrc=sin(2*PI*440*t):d=180:sample_rate=44100" \
+	-c:a mp2fixed -b:a 128k \
+	-vn -y $(TARGET_PATH)/tests/data/hls_sched_a.mka 2>/dev/null
+
+tests/data/hls_sched_extaudio.m3u8: TAG = GEN
+tests/data/hls_sched_extaudio.m3u8: ffmpeg$(PROGSSUF)$(EXESUF) tests/data/hls_sched_v.mkv tests/data/hls_sched_a.mka | tests/data
+	$(M)$(TARGET_EXEC) $(TARGET_PATH)/$< -nostdin \
+	-ss 120 -noaccurate_seek -i $(TARGET_PATH)/tests/data/hls_sched_v.mkv \
+	-ss 120 -noaccurate_seek -i $(TARGET_PATH)/tests/data/hls_sched_a.mka \
+	-ss 120 -map 0:v -map 1:a -c:v mpeg2video -flags +bitexact \
+	-c:a copy -copyts -avoid_negative_ts disabled \
+	-f hls -hls_time 2 -hls_list_size 0 \
+	-hls_segment_filename $(TARGET_PATH)/tests/data/hls_sched_extaudio_%d.ts \
+	$(TARGET_PATH)/tests/data/hls_sched_extaudio.m3u8 2>/dev/null
+
+FATE_HLSENC_FFMPEG_FFPROBE-$(call ALLYES, TESTSRC2_FILTER AEVALSRC_FILTER ARESAMPLE_FILTER LAVFI_INDEV \
+    MPEG4_ENCODER MPEG2VIDEO_ENCODER MP2FIXED_ENCODER \
+    WRAPPED_AVFRAME_DECODER MPEG4_DECODER MPEG2VIDEO_DECODER MP2_DECODER \
+    $(if $(HAVE_BIGENDIAN),PCM_F64BE_DECODER,PCM_F64LE_DECODER) \
+    MPEG4VIDEO_PARSER MPEGVIDEO_PARSER MPEGAUDIO_PARSER \
+    MATROSKA_MUXER MATROSKA_AUDIO_MUXER MATROSKA_DEMUXER MPEGTS_DEMUXER \
+    HLS_MUXER MPEGTS_MUXER FILE_PROTOCOL) += fate-hls-sched-extaudio
+fate-hls-sched-extaudio: tests/data/hls_sched_extaudio.m3u8
+fate-hls-sched-extaudio: CLEANFILES = tests/data/hls_sched_extaudio.m3u8 tests/data/hls_sched_extaudio_*.ts tests/data/hls_sched_v.mkv tests/data/hls_sched_a.mka
+# Probe the first HLS segment's audio stream parameters. Without the fix,
+# audio is starved during the segment's window and only sample_rate=0,
+# channels=0 are reported (the MPEG-TS PSI declares the codec but no audio
+# packets are present). With the fix, audio reaches the segment in time.
+fate-hls-sched-extaudio: CMD = run ffprobe$(PROGSSUF)$(EXESUF) -bitexact -threads $(THREADS) \
+    -analyzeduration 10000000 -probesize 10000000 \
+    -show_entries stream=codec_type,codec_name,sample_rate,channels \
+    -print_format default=nw=1 \
+    $(TARGET_PATH)/tests/data/hls_sched_extaudio_0.ts
+fate-hls-sched-extaudio: CMP = diff
+
 FATE_HLSENC_LAVFI-yes := $(if $(call FRAMECRC), $(FATE_HLSENC_LAVFI-yes))
 
+FATE_FFMPEG_FFPROBE += $(FATE_HLSENC_FFMPEG_FFPROBE-yes)
+
 FATE_FFMPEG += $(FATE_HLSENC_LAVFI-yes)
-fate-hlsenc: $(FATE_HLSENC-yes) $(FATE_HLSENC_PROBE-yes) $(FATE_HLSENC_LAVFI-yes)
+fate-hlsenc: $(FATE_HLSENC-yes) $(FATE_HLSENC_PROBE-yes) $(FATE_HLSENC_LAVFI-yes) $(FATE_HLSENC_FFMPEG_FFPROBE-yes)

tests/ref/fate/hls-sched-extaudio

@@ -0,0 +1,6 @@
+codec_name=mpeg2video
+codec_type=video
+codec_name=mp2
+codec_type=audio
+sample_rate=44100
+channels=1