[PATCH] Add optional "mp4_exact_start" nginx config off/on to show video between keyframes
Roman Arutyunyan
arut at nginx.com
Mon Jun 28 09:53:20 UTC 2021
Hi Tracey,
On Tue, Jun 15, 2021 at 03:49:48PM -0700, Tracey Jaquith wrote:
> # HG changeset patch
> # User Tracey Jaquith <tracey at archive.org>
> # Date 1623797180 0
> # Tue Jun 15 22:46:20 2021 +0000
> # Node ID 1879d49fe0cf739f48287b5a38a83d3a1adab939
> # Parent 5f765427c17ac8cf753967387562201cf4f78dc4
> Add optional "mp4_exact_start" nginx config off/on to show video between keyframes.
I've been thinking about a better name for this, but came up with nothing so
far. I feel like this name does not give the right clue to the user.
Moreover, when this feature is on, the start is not quite "exact", but shifted
a few milliseconds into the past.
> archive.org has been using mod_h264_streaming with a similar "exact start" patch from me since 2013.
> We just moved to nginx mp4 module and are using this patch.
> The technique is to find the video keyframe just before the desired "start" time, and send
> that down the wire so video playback can start immediately.
> Next calculate how many video samples are between the keyframe and desired "start" time
> and update the STTS atom where those samples move the duration from (typically) 1001 to 1.
> This way, initial unwanted video frames play at ~1/30,000s -- so visually the
> video & audio start playing immediately.
>
> You can see an example before/after here (nginx binary built with mp4 module + patch):
>
> https://pi.archive.org/0/items/CSPAN_20160425_022500_2011_White_House_Correspondents_Dinner.mp4?start=12&end=30
> https://pi.archive.org/0/items/CSPAN_20160425_022500_2011_White_House_Correspondents_Dinner.mp4?start=12&end=30&exact=1
>
> Tested on linux and macosx.
>
> (this is me: https://github.com/traceypooh )
We have a few rules about patches and commit messages like 67-character limit
for the first line etc:
http://nginx.org/en/docs/contributing_changes.html
> diff -r 5f765427c17a -r 1879d49fe0cf src/http/modules/ngx_http_mp4_module.c
> --- a/src/http/modules/ngx_http_mp4_module.c Tue Jun 01 17:37:51 2021 +0300
> +++ b/src/http/modules/ngx_http_mp4_module.c Tue Jun 15 22:46:20 2021 +0000
> @@ -43,6 +43,7 @@
> typedef struct {
> size_t buffer_size;
> size_t max_buffer_size;
> + ngx_flag_t exact_start;
> } ngx_http_mp4_conf_t;
>
>
> @@ -340,6 +341,13 @@
> offsetof(ngx_http_mp4_conf_t, max_buffer_size),
> NULL },
>
> + { ngx_string("mp4_exact_start"),
> + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1,
NGX_CONF_TAKE1 -> NGX_CONF_FLAG
> + ngx_conf_set_flag_slot,
> + NGX_HTTP_LOC_CONF_OFFSET,
> + offsetof(ngx_http_mp4_conf_t, exact_start),
> + NULL },
> +
> ngx_null_command
> };
>
> @@ -2156,6 +2164,83 @@
>
>
> static ngx_int_t
> +ngx_http_mp4_exact_start_video(ngx_http_mp4_file_t *mp4, ngx_http_mp4_trak_t *trak)
> +{
> + uint32_t n, speedup_samples, current_count;
> + ngx_uint_t sample_keyframe, start_sample_exact;
> + ngx_mp4_stts_entry_t *entry, *entries_array;
> + ngx_buf_t *data;
> +
> + data = trak->out[NGX_HTTP_MP4_STTS_DATA].buf;
> +
> + // Find the keyframe just before the desired start time - so that we can emit an mp4
> + // where the first frame is a keyframe. We'll "speed up" the first frames to 1000x
> + // normal speed (typically), so they won't be noticed. But this way, perceptively,
> + // playback of the _video_ track can start immediately
> + // (and not have to wait until the keyframe _after_ the desired starting time frame).
> + start_sample_exact = trak->start_sample;
> + for (n = 0; n < trak->sync_samples_entries; n++) {
> + // each element of array is the sample number of a keyframe
> + // sync samples starts from 1 -- so subtract 1
> + sample_keyframe = ngx_mp4_get_32value(trak->stss_data_buf.pos + (n * 4)) - 1;
This can be simplified by introducing entry/end variables like we usually do.
Also, we don't access trak->stss_data_buf directly, but prefer
trak->out[NGX_HTTP_MP4_STSS_ATOM].buf.
ngx_http_mp4_crop_stss_data() provides an example of iterating over stss atom.
> + if (sample_keyframe <= trak->start_sample) {
> + start_sample_exact = sample_keyframe;
> + }
> + if (sample_keyframe >= trak->start_sample) {
> + break;
> + }
> + }
> +
> + if (start_sample_exact < trak->start_sample) {
> + // We're going to prepend an entry with duration=1 for the frames we want to "not see".
> + // MOST of the time (eg: constant video framerate),
> + // we're taking a single element entry array and making it two.
> + speedup_samples = trak->start_sample - start_sample_exact;
> +
> + ngx_log_debug3(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0,
> + "exact trak start_sample move %l to %l (speed up %d samples)\n",
> + trak->start_sample, start_sample_exact, speedup_samples);
> +
> + entries_array = ngx_palloc(mp4->request->pool,
> + (1 + trak->time_to_sample_entries) * sizeof(ngx_mp4_stts_entry_t));
> + if (entries_array == NULL) {
> + return NGX_ERROR;
> + }
> + entry = &(entries_array[1]);
> + ngx_memcpy(entry, (ngx_mp4_stts_entry_t *)data->pos,
> + trak->time_to_sample_entries * sizeof(ngx_mp4_stts_entry_t));
This reallocation can be avoided. Look at NGX_HTTP_MP4_STSC_START buffer
as an example of that. A new 1-element optional buffer NGX_HTTP_MP4_STTS_START
can be introduced right before the stts atom data.
> + current_count = ngx_mp4_get_32value(entry->count);
> + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0,
> + "exact split in 2 video STTS entry from count:%d", current_count);
> +
> + if (current_count <= speedup_samples) {
> + return NGX_ERROR;
> + }
> +
> + ngx_mp4_set_32value(entry->count, current_count - speedup_samples);
> + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0,
> + "exact split new[1]: count:%d duration:%d",
> + ngx_mp4_get_32value(entry->count),
> + ngx_mp4_get_32value(entry->duration));
> + entry--;
> + ngx_mp4_set_32value(entry->count, speedup_samples);
> + ngx_mp4_set_32value(entry->duration, 1);
> + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0,
> + "exact split new[0]: count:%d duration:1",
> + ngx_mp4_get_32value(entry->count));
> +
> + data->pos = (u_char *) entry;
> + trak->time_to_sample_entries++;
> + trak->start_sample = start_sample_exact;
> + data->last = (u_char *) (entry + trak->time_to_sample_entries);
> + }
> +
> + return NGX_OK;
> +}
> +
> +
> +static ngx_int_t
> ngx_http_mp4_crop_stts_data(ngx_http_mp4_file_t *mp4,
> ngx_http_mp4_trak_t *trak, ngx_uint_t start)
> {
> @@ -2164,6 +2249,8 @@
> ngx_buf_t *data;
> ngx_uint_t start_sample, entries, start_sec;
> ngx_mp4_stts_entry_t *entry, *end;
> + ngx_http_mp4_conf_t *conf;
> +
No need for a new empty line here.
> if (start) {
> start_sec = mp4->start;
> @@ -2238,6 +2325,10 @@
> "start_sample:%ui, new count:%uD",
> trak->start_sample, count - rest);
>
> + conf = ngx_http_get_module_loc_conf(mp4->request, ngx_http_mp4_module);
> + if (conf->exact_start) {
> + ngx_http_mp4_exact_start_video(mp4, trak);
> + }
> } else {
> ngx_mp4_set_32value(entry->count, rest);
> data->last = (u_char *) (entry + 1);
> @@ -3590,6 +3681,7 @@
>
> conf->buffer_size = NGX_CONF_UNSET_SIZE;
> conf->max_buffer_size = NGX_CONF_UNSET_SIZE;
> + conf->exact_start = NGX_CONF_UNSET;
This is not enough, a merge is needed too.
>
> return conf;
> }
> _______________________________________________
> nginx-devel mailing list
> nginx-devel at nginx.org
> http://mailman.nginx.org/mailman/listinfo/nginx-devel
I've made a POC patch which incorporates the issues I've mentioned.
I didn't test is properly and the directive name is still not perfect.
--
Roman Arutyunyan
-------------- next part --------------
# HG changeset patch
# User Roman Arutyunyan <arut at nginx.com>
# Date 1623416525 -10800
# Fri Jun 11 16:02:05 2021 +0300
# Node ID 4c7512abda4b335309e3c03d98e8ab680d6dbdd7
# Parent 1ebd78df4ce7262967c5dadce7bac454c4086896
Mp4: mp4_seek_key_frame directive.
The directive enables including all frames from start time to the most recent
key frame in the result. Those frames get the smallest time intervals between
them.
Based on a patch by Tracey Jaquith.
diff --git a/src/http/modules/ngx_http_mp4_module.c b/src/http/modules/ngx_http_mp4_module.c
--- a/src/http/modules/ngx_http_mp4_module.c
+++ b/src/http/modules/ngx_http_mp4_module.c
@@ -21,21 +21,22 @@
#define NGX_HTTP_MP4_STBL_ATOM 9
#define NGX_HTTP_MP4_STSD_ATOM 10
#define NGX_HTTP_MP4_STTS_ATOM 11
-#define NGX_HTTP_MP4_STTS_DATA 12
-#define NGX_HTTP_MP4_STSS_ATOM 13
-#define NGX_HTTP_MP4_STSS_DATA 14
-#define NGX_HTTP_MP4_CTTS_ATOM 15
-#define NGX_HTTP_MP4_CTTS_DATA 16
-#define NGX_HTTP_MP4_STSC_ATOM 17
-#define NGX_HTTP_MP4_STSC_START 18
-#define NGX_HTTP_MP4_STSC_DATA 19
-#define NGX_HTTP_MP4_STSC_END 20
-#define NGX_HTTP_MP4_STSZ_ATOM 21
-#define NGX_HTTP_MP4_STSZ_DATA 22
-#define NGX_HTTP_MP4_STCO_ATOM 23
-#define NGX_HTTP_MP4_STCO_DATA 24
-#define NGX_HTTP_MP4_CO64_ATOM 25
-#define NGX_HTTP_MP4_CO64_DATA 26
+#define NGX_HTTP_MP4_STTS_START 12
+#define NGX_HTTP_MP4_STTS_DATA 13
+#define NGX_HTTP_MP4_STSS_ATOM 14
+#define NGX_HTTP_MP4_STSS_DATA 15
+#define NGX_HTTP_MP4_CTTS_ATOM 16
+#define NGX_HTTP_MP4_CTTS_DATA 17
+#define NGX_HTTP_MP4_STSC_ATOM 18
+#define NGX_HTTP_MP4_STSC_START 19
+#define NGX_HTTP_MP4_STSC_DATA 20
+#define NGX_HTTP_MP4_STSC_END 21
+#define NGX_HTTP_MP4_STSZ_ATOM 22
+#define NGX_HTTP_MP4_STSZ_DATA 23
+#define NGX_HTTP_MP4_STCO_ATOM 24
+#define NGX_HTTP_MP4_STCO_DATA 25
+#define NGX_HTTP_MP4_CO64_ATOM 26
+#define NGX_HTTP_MP4_CO64_DATA 27
#define NGX_HTTP_MP4_LAST_ATOM NGX_HTTP_MP4_CO64_DATA
@@ -43,6 +44,7 @@
typedef struct {
size_t buffer_size;
size_t max_buffer_size;
+ ngx_flag_t seek_key_frame;
} ngx_http_mp4_conf_t;
@@ -54,6 +56,12 @@ typedef struct {
typedef struct {
+ u_char count[4];
+ u_char duration[4];
+} ngx_mp4_stts_entry_t;
+
+
+typedef struct {
uint32_t timescale;
uint32_t time_to_sample_entries;
uint32_t sample_to_chunk_entries;
@@ -95,6 +103,7 @@ typedef struct {
ngx_buf_t stbl_atom_buf;
ngx_buf_t stsd_atom_buf;
ngx_buf_t stts_atom_buf;
+ ngx_buf_t stts_start_buf;
ngx_buf_t stts_data_buf;
ngx_buf_t stss_atom_buf;
ngx_buf_t stss_data_buf;
@@ -111,6 +120,7 @@ typedef struct {
ngx_buf_t co64_atom_buf;
ngx_buf_t co64_data_buf;
+ ngx_mp4_stts_entry_t stts_start_entry;
ngx_mp4_stsc_entry_t stsc_start_chunk_entry;
ngx_mp4_stsc_entry_t stsc_end_chunk_entry;
} ngx_http_mp4_trak_t;
@@ -277,6 +287,8 @@ static ngx_int_t ngx_http_mp4_update_stt
ngx_http_mp4_trak_t *trak);
static ngx_int_t ngx_http_mp4_crop_stts_data(ngx_http_mp4_file_t *mp4,
ngx_http_mp4_trak_t *trak, ngx_uint_t start);
+static ngx_int_t ngx_http_mp4_rewind_stts_data(ngx_http_mp4_file_t *mp4,
+ ngx_http_mp4_trak_t *trak);
static ngx_int_t ngx_http_mp4_read_stss_atom(ngx_http_mp4_file_t *mp4,
uint64_t atom_data_size);
static ngx_int_t ngx_http_mp4_update_stss_atom(ngx_http_mp4_file_t *mp4,
@@ -340,6 +352,13 @@ static ngx_command_t ngx_http_mp4_comma
offsetof(ngx_http_mp4_conf_t, max_buffer_size),
NULL },
+ { ngx_string("mp4_seek_key_frame"),
+ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG,
+ ngx_conf_set_flag_slot,
+ NGX_HTTP_LOC_CONF_OFFSET,
+ offsetof(ngx_http_mp4_conf_t, seek_key_frame),
+ NULL },
+
ngx_null_command
};
@@ -2040,11 +2059,6 @@ typedef struct {
u_char entries[4];
} ngx_mp4_stts_atom_t;
-typedef struct {
- u_char count[4];
- u_char duration[4];
-} ngx_mp4_stts_entry_t;
-
static ngx_int_t
ngx_http_mp4_read_stts_atom(ngx_http_mp4_file_t *mp4, uint64_t atom_data_size)
@@ -2140,10 +2154,20 @@ ngx_http_mp4_update_stts_atom(ngx_http_m
return NGX_ERROR;
}
+ if (ngx_http_mp4_rewind_stts_data(mp4, trak) != NGX_OK) {
+ return NGX_ERROR;
+ }
+
ngx_log_debug1(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0,
"time-to-sample entries:%uD", trak->time_to_sample_entries);
atom_size = sizeof(ngx_mp4_stts_atom_t) + (data->last - data->pos);
+
+ data = trak->out[NGX_HTTP_MP4_STTS_START].buf;
+ if (data) {
+ atom_size += data->last - data->pos;
+ }
+
trak->size += atom_size;
atom = trak->out[NGX_HTTP_MP4_STTS_ATOM].buf;
@@ -2253,6 +2277,69 @@ found:
}
+static ngx_int_t
+ngx_http_mp4_rewind_stts_data(ngx_http_mp4_file_t *mp4,
+ ngx_http_mp4_trak_t *trak)
+{
+ uint32_t start_sample, rewind, sample, *entry, *end;
+ ngx_buf_t *data;
+ ngx_http_mp4_conf_t *conf;
+ ngx_mp4_stts_entry_t *rewind_entry;
+
+ conf = ngx_http_get_module_loc_conf(mp4->request, ngx_http_mp4_module);
+ if (!conf->seek_key_frame) {
+ return NGX_OK;
+ }
+
+ data = trak->out[NGX_HTTP_MP4_STSS_DATA].buf;
+ if (data == NULL) {
+ return NGX_OK;
+ }
+
+ entry = (uint32_t *) data->pos;
+ end = (uint32_t *) data->last;
+
+ /* sync samples starts from 1 */
+ start_sample = trak->start_sample + 1;
+
+ rewind = 0;
+
+ while (entry < end) {
+ sample = ngx_mp4_get_32value(entry);
+ if (sample > start_sample) {
+ break;
+ }
+
+ rewind = start_sample - sample;
+ entry++;
+ }
+
+ if (rewind == 0) {
+ return NGX_OK;
+ }
+
+ trak->time_to_sample_entries++;
+ trak->start_sample -= rewind;
+
+ rewind_entry = &trak->stts_start_entry;
+ ngx_mp4_set_32value(rewind_entry->count, rewind);
+ ngx_mp4_set_32value(rewind_entry->duration, 1);
+
+ data = &trak->stts_start_buf;
+ data->temporary = 1;
+ data->pos = (u_char *) rewind_entry;
+ data->last = (u_char *) rewind_entry + sizeof(ngx_mp4_stts_entry_t);
+
+ trak->out[NGX_HTTP_MP4_STTS_START].buf = data;
+
+ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0,
+ "mp4 rewind samples:%uD, start_sample:%ui",
+ rewind, trak->start_sample);
+
+ return NGX_OK;
+}
+
+
typedef struct {
u_char size[4];
u_char name[4];
@@ -3590,6 +3677,7 @@ ngx_http_mp4_create_conf(ngx_conf_t *cf)
conf->buffer_size = NGX_CONF_UNSET_SIZE;
conf->max_buffer_size = NGX_CONF_UNSET_SIZE;
+ conf->seek_key_frame = NGX_CONF_UNSET;
return conf;
}
@@ -3604,6 +3692,7 @@ ngx_http_mp4_merge_conf(ngx_conf_t *cf,
ngx_conf_merge_size_value(conf->buffer_size, prev->buffer_size, 512 * 1024);
ngx_conf_merge_size_value(conf->max_buffer_size, prev->max_buffer_size,
10 * 1024 * 1024);
+ ngx_conf_merge_value(conf->seek_key_frame, prev->seek_key_frame, 0);
return NGX_CONF_OK;
}
More information about the nginx-devel
mailing list