Re: Ошибки при использовании zlib-ng

raven_kg на megaline.kg raven_kg на megaline.kg
Пн Мар 29 07:00:21 UTC 2021


29.03.2021 08:09, Maxim Dounin пишет:
> Hello!
>
> On Fri, Mar 26, 2021 at 09:54:43PM +0300, Maxim Dounin wrote:
>
>> Hello!
>>
>> On Fri, Mar 26, 2021 at 01:32:48PM +0300, Sergey Kandaurov wrote:
>>
>>>> On 26 Mar 2021, at 13:14, raven_kg на megaline.kg wrote:
>>>>
>>>> После смены нативной zlib-1.2.7 на zlib-ng 2.0.1 (собрана в режиме совместимости с zlib) лог буквально завален ошибками:
>>>>
>>>> "gzip filter failed to use preallocated memory: 65536 of 0 while sending to client"
>>>>
>>>> Хотя, насколько я припоминаю, обход ошибок такого рода упоминался в анонсе к какой-то из версий 1.13.
>>>>
>>> Попробуйте патч, при сборке с zlib-ng:
>>>
>>> diff --git a/src/http/modules/ngx_http_gzip_filter_module.c b/src/http/modules/ngx_http_gzip_filter_module.c
>>> --- a/src/http/modules/ngx_http_gzip_filter_module.c
>>> +++ b/src/http/modules/ngx_http_gzip_filter_module.c
>>> @@ -516,7 +516,7 @@ ngx_http_gzip_filter_memory(ngx_http_req
>>>            */
>>>   
>>>           if (conf->level == 1) {
>>> -            wbits = ngx_max(wbits, 13);
>>> +            wbits = ngx_max(wbits, 15);
>> Насколько я вижу, в zlib-ng используюся те же 13, что и в варианте
>> от Intel:
>>
>> https://github.com/jtkukunas/zlib/blob/master/deflate.c#L296
>> https://github.com/zlib-ng/zlib-ng/blob/develop/deflate.c#L304
>>
>> А вот аллокация под hash стала 2x64k.
>>
>> (В интеловском варианте, кстати, за последнее время и hash
>> подужался, и windowBits в 13 ставится только для значений, больших
>> 13.  Возможно, на него стоит ещё разок взглянуть и урезать осетра.)
> Взглянул, урезал.
>
> # HG changeset patch
> # User Maxim Dounin <mdounin на mdounin.ru>
> # Date 1616983280 -10800
> #      Mon Mar 29 05:01:20 2021 +0300
> # Node ID d82e9d8285e2552cafcea2033069cdd31f4a5d32
> # Parent  1ebd78df4ce7262967c5dadce7bac454c4086896
> Gzip: support for zlib-ng.
>
> diff --git a/src/http/modules/ngx_http_gzip_filter_module.c b/src/http/modules/ngx_http_gzip_filter_module.c
> --- a/src/http/modules/ngx_http_gzip_filter_module.c
> +++ b/src/http/modules/ngx_http_gzip_filter_module.c
> @@ -57,6 +57,7 @@ typedef struct {
>       unsigned             nomem:1;
>       unsigned             buffering:1;
>       unsigned             intel:1;
> +    unsigned             zlib_ng:1;
>   
>       size_t               zin;
>       size_t               zout;
> @@ -214,6 +215,7 @@ static ngx_http_output_header_filter_pt
>   static ngx_http_output_body_filter_pt    ngx_http_next_body_filter;
>   
>   static ngx_uint_t  ngx_http_gzip_assume_intel;
> +static ngx_uint_t  ngx_http_gzip_assume_zlib_ng;
>   
>   
>   static ngx_int_t
> @@ -506,7 +508,7 @@ ngx_http_gzip_filter_memory(ngx_http_req
>       if (!ngx_http_gzip_assume_intel) {
>           ctx->allocated = 8192 + (1 << (wbits + 2)) + (1 << (memlevel + 9));
>   
> -    } else {
> +    } else if (!ngx_http_gzip_assume_zlib_ng) {
>           /*
>            * A zlib variant from Intel, https://github.com/jtkukunas/zlib.
>            * It can force window bits to 13 for fast compression level,
> @@ -523,6 +525,20 @@ ngx_http_gzip_filter_memory(ngx_http_req
>                            + (1 << (ngx_max(memlevel, 8) + 8))
>                            + (1 << (memlevel + 8));
>           ctx->intel = 1;
> +
> +    } else {
> +        /*
> +         * Another zlib variant, https://github.com/zlib-ng/zlib-ng.
> +         * Similar to Intel's variant, though uses 128K hash.
> +         */
> +
> +        if (conf->level == 1) {
> +            wbits = ngx_max(wbits, 13);
> +        }
> +
> +        ctx->allocated = 8192 + 16 + (1 << (wbits + 2))
> +                         + 131072 + (1 << (memlevel + 8));
> +        ctx->zlib_ng = 1;
>       }
>   }
>   
> @@ -945,11 +961,14 @@ ngx_http_gzip_filter_alloc(void *opaque,
>           return p;
>       }
>   
> -    if (ctx->intel) {
> +    if (ctx->zlib_ng) {
>           ngx_log_error(NGX_LOG_ALERT, ctx->request->connection->log, 0,
>                         "gzip filter failed to use preallocated memory: "
>                         "%ud of %ui", items * size, ctx->allocated);
>   
> +    } else if (ctx->intel) {
> +        ngx_http_gzip_assume_zlib_ng = 1;
> +
>       } else {
>           ngx_http_gzip_assume_intel = 1;
>       }
> # HG changeset patch
> # User Maxim Dounin <mdounin на mdounin.ru>
> # Date 1616983654 -10800
> #      Mon Mar 29 05:07:34 2021 +0300
> # Node ID e70b352ddb2ed1e92997408b3926e64aa55b4a14
> # Parent  d82e9d8285e2552cafcea2033069cdd31f4a5d32
> Gzip: updated handling of zlib variant from Intel.
>
> In current versions (all versions based on zlib 1.2.11, at least
> since 2018) it no longer uses 64K hash and does not force window
> bits to 13 if it is less than 13.  That is, it needs just 16 bytes
> more memory than normal zlib, so these bytes are simply added to
> the normal size calculation.
>
> diff --git a/src/http/modules/ngx_http_gzip_filter_module.c b/src/http/modules/ngx_http_gzip_filter_module.c
> --- a/src/http/modules/ngx_http_gzip_filter_module.c
> +++ b/src/http/modules/ngx_http_gzip_filter_module.c
> @@ -56,7 +56,6 @@ typedef struct {
>       unsigned             done:1;
>       unsigned             nomem:1;
>       unsigned             buffering:1;
> -    unsigned             intel:1;
>       unsigned             zlib_ng:1;
>   
>       size_t               zin;
> @@ -214,7 +213,6 @@ static ngx_str_t  ngx_http_gzip_ratio =
>   static ngx_http_output_header_filter_pt  ngx_http_next_header_filter;
>   static ngx_http_output_body_filter_pt    ngx_http_next_body_filter;
>   
> -static ngx_uint_t  ngx_http_gzip_assume_intel;
>   static ngx_uint_t  ngx_http_gzip_assume_zlib_ng;
>   
>   
> @@ -503,33 +501,21 @@ ngx_http_gzip_filter_memory(ngx_http_req
>        * 8K is for zlib deflate_state, it takes
>        *  *) 5816 bytes on i386 and sparc64 (32-bit mode)
>        *  *) 5920 bytes on amd64 and sparc64
> +     *
> +     * A zlib variant from Intel (https://github.com/jtkukunas/zlib)
> +     * uses additional 16-byte padding in one of window-sized buffers.
>        */
>   
> -    if (!ngx_http_gzip_assume_intel) {
> -        ctx->allocated = 8192 + (1 << (wbits + 2)) + (1 << (memlevel + 9));
> -
> -    } else if (!ngx_http_gzip_assume_zlib_ng) {
> -        /*
> -         * A zlib variant from Intel, https://github.com/jtkukunas/zlib.
> -         * It can force window bits to 13 for fast compression level,
> -         * on processors with SSE 4.2 it uses 64K hash instead of scaling
> -         * it from the specified memory level, and also introduces
> -         * 16-byte padding in one out of the two window-sized buffers.
> -         */
> -
> -        if (conf->level == 1) {
> -            wbits = ngx_max(wbits, 13);
> -        }
> -
> +    if (!ngx_http_gzip_assume_zlib_ng) {
>           ctx->allocated = 8192 + 16 + (1 << (wbits + 2))
> -                         + (1 << (ngx_max(memlevel, 8) + 8))
> -                         + (1 << (memlevel + 8));
> -        ctx->intel = 1;
> +                         + (1 << (memlevel + 9));
>   
>       } else {
>           /*
>            * Another zlib variant, https://github.com/zlib-ng/zlib-ng.
> -         * Similar to Intel's variant, though uses 128K hash.
> +         * It forces window bits to 13 for fast compression level,
> +         * uses 16-byte padding in one of window-sized buffers, and
> +         * uses 128K hash.
>            */
>   
>           if (conf->level == 1) {
> @@ -966,11 +952,8 @@ ngx_http_gzip_filter_alloc(void *opaque,
>                         "gzip filter failed to use preallocated memory: "
>                         "%ud of %ui", items * size, ctx->allocated);
>   
> -    } else if (ctx->intel) {
> +    } else {
>           ngx_http_gzip_assume_zlib_ng = 1;
> -
> -    } else {
> -        ngx_http_gzip_assume_intel = 1;
>       }
>   
>       p = ngx_palloc(ctx->request->pool, items * size);
>
Будут-ли эти правки включены в какой-либо из релизов?



Подробная информация о списке рассылки nginx-ru