[PATCH] Add optional "&exact=1" CGI arg to show video between keyframes
Tracey Jaquith
tracey at archive.org
Thu Jun 3 19:54:49 UTC 2021
# HG changeset patch
# User Tracey Jaquith <tracey at archive.org>
# Date 1622678642 0
# Thu Jun 03 00:04:02 2021 +0000
# Node ID 5da9c62fa61016600f2c59982ae184e2811be427
# Parent 5f765427c17ac8cf753967387562201cf4f78dc4
Add optional "&exact=1" CGI arg to show video between keyframes.
archive.org has been using mod_h264_streaming with an "&exact=1" patch from me since 2013.
We just moved to nginx mp4 module and are using this patch.
The technique is to find the keyframe just before the desired "start" time, and send
that down the wire so video playback can start immediately.
Next calculate how many video samples are between the keyframe and desired "start" time
and update the STTS atom where those samples move the duration from (typically) 1001 to 1.
This way, initial unwanted video frames play at ~1/30,000s -- so visually the
video & audio start playing immediately.
You can see an example before/after here (nginx binary built with mp4 module + patch):
https://pi.archive.org/0/items/CSPAN_20160425_022500_2011_White_House_Correspondents_Dinner.mp4?start=12&end=30
https://pi.archive.org/0/items/CSPAN_20160425_022500_2011_White_House_Correspondents_Dinner.mp4?start=12&end=30&exact=1
Tested on linux and macosx.
I realize two var declarations should style-wise get moved "up" in the lower hooked in method.
However, to minimize the changeset/patch, I figured we should at least start here and see what you folks think.
(this is me: https://github.com/traceypooh )
diff -r 5f765427c17a -r 5da9c62fa610 src/http/modules/ngx_http_mp4_module.c
--- a/src/http/modules/ngx_http_mp4_module.c Tue Jun 01 17:37:51 2021 +0300
+++ b/src/http/modules/ngx_http_mp4_module.c Thu Jun 03 00:04:02 2021 +0000
@@ -2045,6 +2045,109 @@
u_char duration[4];
} ngx_mp4_stts_entry_t;
+typedef struct {
+ uint32_t speedup_samples;
+ ngx_uint_t speedup_seconds;
+} ngx_mp4_exact_t;
+
+static void
+exact_video_adjustment(ngx_http_mp4_file_t *mp4, ngx_http_mp4_trak_t *trak, ngx_mp4_exact_t *exact)
+{
+ // will parse STTS -- time-to-sample atom
+ ngx_str_t value;
+ ngx_buf_t *stts_data;
+ ngx_buf_t *atom;
+ ngx_mp4_stts_entry_t *stts_entry, *stts_end;
+ uint32_t count, duration, j, n, sample_keyframe, sample_num;
+ uint64_t sample_time, seconds, start_seconds_closest_keyframe;
+ uint8_t is_keyframe;
+
+ exact->speedup_samples = 0;
+ exact->speedup_seconds = 0;
+
+ // if '&exact=1' CGI arg isn't present, do nothing
+ if (!(ngx_http_arg(mp4->request, (u_char *) "exact", 5, &value) == NGX_OK)) {
+ return;
+ }
+
+ if (!trak->sync_samples_entries) {
+ // Highly unlikely video STSS got parsed and _every_ sample is a keyframe.
+ // However, if the case, we don't need to adjust the video at all.
+ return;
+ }
+
+ // check HDLR atom to see if this trak is video or audio
+ atom = trak->out[NGX_HTTP_MP4_HDLR_ATOM].buf;
+ // 'vide' or 'soun'
+ if (!(atom->pos[16] == 'v' &&
+ atom->pos[17] == 'i' &&
+ atom->pos[18] == 'd' &&
+ atom->pos[19] == 'e')) {
+ return; // do nothing if not video
+ }
+
+
+ stts_data = trak->out[NGX_HTTP_MP4_STTS_DATA].buf;
+ stts_entry = (ngx_mp4_stts_entry_t *) stts_data->pos;
+ stts_end = (ngx_mp4_stts_entry_t *) stts_data->last;
+
+ sample_num = 0; // they start at one <shrug>
+ sample_time = 0;
+ start_seconds_closest_keyframe = 0;
+ while (stts_entry < stts_end) {
+ // STTS === time-to-sample atom
+ // each entry is 4B and [sample count][sample duration] (since durations can vary)
+ count = ngx_mp4_get_32value(stts_entry->count);
+ duration = ngx_mp4_get_32value(stts_entry->duration);
+
+ for (j = 0; j < count; j++) {
+ sample_num++;
+
+ // search STSS sync sample entries to see if this sample is a keyframe
+ is_keyframe = (trak->sync_samples_entries ? 0 : 1);
+ for (n = 0; n < trak->sync_samples_entries; n++) {
+ // each one of this these are a video sample number keyframe
+ sample_keyframe = ngx_mp4_get_32value(trak->stss_data_buf.pos + (n * 4));
+ if (sample_keyframe == sample_num) {
+ is_keyframe = 1;
+ break;
+ }
+ if (sample_keyframe > sample_num) {
+ break;
+ }
+ }
+
+ seconds = sample_time * 1000 / trak->timescale;
+ sample_time += duration;
+
+ if (seconds > mp4->start) {
+ goto found;
+ }
+
+ if (is_keyframe) {
+ start_seconds_closest_keyframe = seconds;
+ exact->speedup_samples = 0;
+ } else {
+ exact->speedup_samples++;
+ }
+ }
+
+ stts_entry++;
+ }
+
+ found:
+
+ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0,
+ "exact_video_adjustment() new keyframe start: %d, speedup first %d samples",
+ start_seconds_closest_keyframe,
+ exact->speedup_samples);
+
+ // NOTE: begin 1 start position before keyframe to ensure first video frame emitted is always
+ // a keyframe
+ exact->speedup_seconds = mp4->start - start_seconds_closest_keyframe
+ - (start_seconds_closest_keyframe ? 1 : 0);
+}
+
static ngx_int_t
ngx_http_mp4_read_stts_atom(ngx_http_mp4_file_t *mp4, uint64_t atom_data_size)
@@ -2164,10 +2267,14 @@
ngx_buf_t *data;
ngx_uint_t start_sample, entries, start_sec;
ngx_mp4_stts_entry_t *entry, *end;
+ ngx_mp4_exact_t exact;
if (start) {
start_sec = mp4->start;
+ exact_video_adjustment(mp4, trak, &exact);
+ start_sec -= exact.speedup_seconds;
+
ngx_log_debug1(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0,
"mp4 stts crop start_time:%ui", start_sec);
@@ -2230,6 +2337,42 @@
if (start) {
ngx_mp4_set_32value(entry->count, count - rest);
+
+ if (exact.speedup_samples) {
+ // We're going to prepend an entry with duration=1 for the frames we want to "not see".
+ // MOST of the time, we're taking a single element entry array and making it two.
+ uint32_t current_count = ngx_mp4_get_32value(entry->count);
+ ngx_mp4_stts_entry_t* entries_array = ngx_palloc(mp4->request->pool,
+ (1 + entries) * sizeof(ngx_mp4_stts_entry_t));
+ if (entries_array == NULL) {
+ return NGX_ERROR;
+ }
+ ngx_copy(&(entries_array[1]), entry, entries * sizeof(ngx_mp4_stts_entry_t));
+
+ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0,
+ "exact split in 2 video STTS entry from count:%d", current_count);
+
+ if (current_count <= exact.speedup_samples)
+ return NGX_ERROR;
+
+ entry = &(entries_array[1]);
+ ngx_mp4_set_32value(entry->count, current_count - exact.speedup_samples);
+ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0,
+ "exact split new[1]: count:%d duration:%d",
+ ngx_mp4_get_32value(entry->count),
+ ngx_mp4_get_32value(entry->duration));
+ entry--;
+ ngx_mp4_set_32value(entry->count, exact.speedup_samples);
+ ngx_mp4_set_32value(entry->duration, 1);
+ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0,
+ "exact split new[0]: count:%d duration:1",
+ ngx_mp4_get_32value(entry->count));
+
+ data->last = (u_char *) (entry + 2);
+
+ entries++;
+ }
+
data->pos = (u_char *) entry;
trak->time_to_sample_entries = entries;
trak->start_sample = start_sample;
More information about the nginx-devel
mailing list