Re: Ошибки при использовании zlib-ng
raven_kg на megaline.kg
raven_kg на megaline.kg
Пн Мар 29 07:00:21 UTC 2021
29.03.2021 08:09, Maxim Dounin пишет:
> Hello!
>
> On Fri, Mar 26, 2021 at 09:54:43PM +0300, Maxim Dounin wrote:
>
>> Hello!
>>
>> On Fri, Mar 26, 2021 at 01:32:48PM +0300, Sergey Kandaurov wrote:
>>
>>>> On 26 Mar 2021, at 13:14, raven_kg на megaline.kg wrote:
>>>>
>>>> После смены нативной zlib-1.2.7 на zlib-ng 2.0.1 (собрана в режиме совместимости с zlib) лог буквально завален ошибками:
>>>>
>>>> "gzip filter failed to use preallocated memory: 65536 of 0 while sending to client"
>>>>
>>>> Хотя, насколько я припоминаю, обход ошибок такого рода упоминался в анонсе к какой-то из версий 1.13.
>>>>
>>> Попробуйте патч, при сборке с zlib-ng:
>>>
>>> diff --git a/src/http/modules/ngx_http_gzip_filter_module.c b/src/http/modules/ngx_http_gzip_filter_module.c
>>> --- a/src/http/modules/ngx_http_gzip_filter_module.c
>>> +++ b/src/http/modules/ngx_http_gzip_filter_module.c
>>> @@ -516,7 +516,7 @@ ngx_http_gzip_filter_memory(ngx_http_req
>>> */
>>>
>>> if (conf->level == 1) {
>>> - wbits = ngx_max(wbits, 13);
>>> + wbits = ngx_max(wbits, 15);
>> Насколько я вижу, в zlib-ng используюся те же 13, что и в варианте
>> от Intel:
>>
>> https://github.com/jtkukunas/zlib/blob/master/deflate.c#L296
>> https://github.com/zlib-ng/zlib-ng/blob/develop/deflate.c#L304
>>
>> А вот аллокация под hash стала 2x64k.
>>
>> (В интеловском варианте, кстати, за последнее время и hash
>> подужался, и windowBits в 13 ставится только для значений, больших
>> 13. Возможно, на него стоит ещё разок взглянуть и урезать осетра.)
> Взглянул, урезал.
>
> # HG changeset patch
> # User Maxim Dounin <mdounin на mdounin.ru>
> # Date 1616983280 -10800
> # Mon Mar 29 05:01:20 2021 +0300
> # Node ID d82e9d8285e2552cafcea2033069cdd31f4a5d32
> # Parent 1ebd78df4ce7262967c5dadce7bac454c4086896
> Gzip: support for zlib-ng.
>
> diff --git a/src/http/modules/ngx_http_gzip_filter_module.c b/src/http/modules/ngx_http_gzip_filter_module.c
> --- a/src/http/modules/ngx_http_gzip_filter_module.c
> +++ b/src/http/modules/ngx_http_gzip_filter_module.c
> @@ -57,6 +57,7 @@ typedef struct {
> unsigned nomem:1;
> unsigned buffering:1;
> unsigned intel:1;
> + unsigned zlib_ng:1;
>
> size_t zin;
> size_t zout;
> @@ -214,6 +215,7 @@ static ngx_http_output_header_filter_pt
> static ngx_http_output_body_filter_pt ngx_http_next_body_filter;
>
> static ngx_uint_t ngx_http_gzip_assume_intel;
> +static ngx_uint_t ngx_http_gzip_assume_zlib_ng;
>
>
> static ngx_int_t
> @@ -506,7 +508,7 @@ ngx_http_gzip_filter_memory(ngx_http_req
> if (!ngx_http_gzip_assume_intel) {
> ctx->allocated = 8192 + (1 << (wbits + 2)) + (1 << (memlevel + 9));
>
> - } else {
> + } else if (!ngx_http_gzip_assume_zlib_ng) {
> /*
> * A zlib variant from Intel, https://github.com/jtkukunas/zlib.
> * It can force window bits to 13 for fast compression level,
> @@ -523,6 +525,20 @@ ngx_http_gzip_filter_memory(ngx_http_req
> + (1 << (ngx_max(memlevel, 8) + 8))
> + (1 << (memlevel + 8));
> ctx->intel = 1;
> +
> + } else {
> + /*
> + * Another zlib variant, https://github.com/zlib-ng/zlib-ng.
> + * Similar to Intel's variant, though uses 128K hash.
> + */
> +
> + if (conf->level == 1) {
> + wbits = ngx_max(wbits, 13);
> + }
> +
> + ctx->allocated = 8192 + 16 + (1 << (wbits + 2))
> + + 131072 + (1 << (memlevel + 8));
> + ctx->zlib_ng = 1;
> }
> }
>
> @@ -945,11 +961,14 @@ ngx_http_gzip_filter_alloc(void *opaque,
> return p;
> }
>
> - if (ctx->intel) {
> + if (ctx->zlib_ng) {
> ngx_log_error(NGX_LOG_ALERT, ctx->request->connection->log, 0,
> "gzip filter failed to use preallocated memory: "
> "%ud of %ui", items * size, ctx->allocated);
>
> + } else if (ctx->intel) {
> + ngx_http_gzip_assume_zlib_ng = 1;
> +
> } else {
> ngx_http_gzip_assume_intel = 1;
> }
> # HG changeset patch
> # User Maxim Dounin <mdounin на mdounin.ru>
> # Date 1616983654 -10800
> # Mon Mar 29 05:07:34 2021 +0300
> # Node ID e70b352ddb2ed1e92997408b3926e64aa55b4a14
> # Parent d82e9d8285e2552cafcea2033069cdd31f4a5d32
> Gzip: updated handling of zlib variant from Intel.
>
> In current versions (all versions based on zlib 1.2.11, at least
> since 2018) it no longer uses 64K hash and does not force window
> bits to 13 if it is less than 13. That is, it needs just 16 bytes
> more memory than normal zlib, so these bytes are simply added to
> the normal size calculation.
>
> diff --git a/src/http/modules/ngx_http_gzip_filter_module.c b/src/http/modules/ngx_http_gzip_filter_module.c
> --- a/src/http/modules/ngx_http_gzip_filter_module.c
> +++ b/src/http/modules/ngx_http_gzip_filter_module.c
> @@ -56,7 +56,6 @@ typedef struct {
> unsigned done:1;
> unsigned nomem:1;
> unsigned buffering:1;
> - unsigned intel:1;
> unsigned zlib_ng:1;
>
> size_t zin;
> @@ -214,7 +213,6 @@ static ngx_str_t ngx_http_gzip_ratio =
> static ngx_http_output_header_filter_pt ngx_http_next_header_filter;
> static ngx_http_output_body_filter_pt ngx_http_next_body_filter;
>
> -static ngx_uint_t ngx_http_gzip_assume_intel;
> static ngx_uint_t ngx_http_gzip_assume_zlib_ng;
>
>
> @@ -503,33 +501,21 @@ ngx_http_gzip_filter_memory(ngx_http_req
> * 8K is for zlib deflate_state, it takes
> * *) 5816 bytes on i386 and sparc64 (32-bit mode)
> * *) 5920 bytes on amd64 and sparc64
> + *
> + * A zlib variant from Intel (https://github.com/jtkukunas/zlib)
> + * uses additional 16-byte padding in one of window-sized buffers.
> */
>
> - if (!ngx_http_gzip_assume_intel) {
> - ctx->allocated = 8192 + (1 << (wbits + 2)) + (1 << (memlevel + 9));
> -
> - } else if (!ngx_http_gzip_assume_zlib_ng) {
> - /*
> - * A zlib variant from Intel, https://github.com/jtkukunas/zlib.
> - * It can force window bits to 13 for fast compression level,
> - * on processors with SSE 4.2 it uses 64K hash instead of scaling
> - * it from the specified memory level, and also introduces
> - * 16-byte padding in one out of the two window-sized buffers.
> - */
> -
> - if (conf->level == 1) {
> - wbits = ngx_max(wbits, 13);
> - }
> -
> + if (!ngx_http_gzip_assume_zlib_ng) {
> ctx->allocated = 8192 + 16 + (1 << (wbits + 2))
> - + (1 << (ngx_max(memlevel, 8) + 8))
> - + (1 << (memlevel + 8));
> - ctx->intel = 1;
> + + (1 << (memlevel + 9));
>
> } else {
> /*
> * Another zlib variant, https://github.com/zlib-ng/zlib-ng.
> - * Similar to Intel's variant, though uses 128K hash.
> + * It forces window bits to 13 for fast compression level,
> + * uses 16-byte padding in one of window-sized buffers, and
> + * uses 128K hash.
> */
>
> if (conf->level == 1) {
> @@ -966,11 +952,8 @@ ngx_http_gzip_filter_alloc(void *opaque,
> "gzip filter failed to use preallocated memory: "
> "%ud of %ui", items * size, ctx->allocated);
>
> - } else if (ctx->intel) {
> + } else {
> ngx_http_gzip_assume_zlib_ng = 1;
> -
> - } else {
> - ngx_http_gzip_assume_intel = 1;
> }
>
> p = ngx_palloc(ctx->request->pool, items * size);
>
Будут-ли эти правки включены в какой-либо из релизов?
Подробная информация о списке рассылки nginx-ru