Nginx crash - possibly keepalives
Maxim Dounin
mdounin at mdounin.ru
Fri Oct 14 19:43:45 UTC 2011
Hello!
On Wed, Oct 12, 2011 at 09:22:41PM +0400, Maxim Dounin wrote:
> Hello!
>
> On Wed, Oct 12, 2011 at 09:32:41AM -0700, Matthieu Tourne wrote:
>
> > On Wed, Oct 12, 2011 at 5:20 AM, Maxim Dounin <mdounin at mdounin.ru> wrote:
> >
> > > Hello!
> > >
> > > On Tue, Oct 11, 2011 at 07:53:36PM -0700, Matthieu Tourne wrote:
> > >
> > > > Hi all,
> > > >
> > > > After turning on keepalives, we've been seeing one crash pretty
> > > > consistently.
> > > > We're running nginx 1.1.5 and here is the backtrace :
[...]
> Ok, it looks like I'm right and all buffers are in busy chain.
> Likely this happens due to upstream sending response in many small
> chunks.
>
> I'll try to reproduce it here and provide proper fix.
Please try the attached patch.
Maxim Dounin
-------------- next part --------------
# HG changeset patch
# User Maxim Dounin <mdounin at mdounin.ru>
# Date 1318621020 -14400
# Node ID c91f35091533962be9bb3c6cb64f6acca60ff480
# Parent 1e17fe554aa2de75684f7a3898d8506281c4a323
Event pipe fixes for complex protocols.
1. In ngx_event_pipe_write_chain_to_temp_file() make sure to fully write
all shadow buffers up to last_shadow. With this change recycled buffers
cannot appear in p->out anymore. This also fixes segmentation faults
observed due to ngx_event_pipe_write_chain_to_temp() not freeing any
raw buffers while still returning NGX_OK.
2. In ngx_event_pipe_write_to_downstream() we now properly check for busy
size as a size of buffers, not a size of data in these buffers. This
fixes situations where all available buffers became busy (including
segmentation faults due to this).
3. The ngx_event_pipe_free_shadow_raw_buf() function is dropped. It's
incorrect and not needed.
diff --git a/src/event/ngx_event_pipe.c b/src/event/ngx_event_pipe.c
--- a/src/event/ngx_event_pipe.c
+++ b/src/event/ngx_event_pipe.c
@@ -15,8 +15,6 @@ static ngx_int_t ngx_event_pipe_write_to
static ngx_int_t ngx_event_pipe_write_chain_to_temp_file(ngx_event_pipe_t *p);
static ngx_inline void ngx_event_pipe_remove_shadow_links(ngx_buf_t *buf);
-static ngx_inline void ngx_event_pipe_free_shadow_raw_buf(ngx_chain_t **free,
- ngx_buf_t *buf);
static ngx_int_t ngx_event_pipe_drain_chains(ngx_event_pipe_t *p);
@@ -576,17 +574,13 @@ ngx_event_pipe_write_to_downstream(ngx_e
if (p->out) {
cl = p->out;
- if (cl->buf->recycled
- && bsize + cl->buf->last - cl->buf->pos > p->busy_size)
- {
- flush = 1;
- break;
+ if (cl->buf->recycled) {
+ ngx_log_error(NGX_LOG_ALERT, p->log, 0,
+ "recycled buffer in pipe out chain");
}
p->out = p->out->next;
- ngx_event_pipe_free_shadow_raw_buf(&p->free_raw_bufs, cl->buf);
-
} else if (!p->cacheable && p->in) {
cl = p->in;
@@ -596,24 +590,13 @@ ngx_event_pipe_write_to_downstream(ngx_e
cl->buf->pos,
cl->buf->last - cl->buf->pos);
- if (cl->buf->recycled
- && cl->buf->last_shadow
- && bsize + cl->buf->last - cl->buf->pos > p->busy_size)
- {
- if (!prev_last_shadow) {
- p->in = p->in->next;
-
- cl->next = NULL;
-
- if (out) {
- *ll = cl;
- } else {
- out = cl;
- }
+ if (cl->buf->recycled && prev_last_shadow) {
+ if (bsize + cl->buf->end - cl->buf->start > p->busy_size) {
+ flush = 1;
+ break;
}
- flush = 1;
- break;
+ bsize += cl->buf->end - cl->buf->start;
}
prev_last_shadow = cl->buf->last_shadow;
@@ -624,10 +607,6 @@ ngx_event_pipe_write_to_downstream(ngx_e
break;
}
- if (cl->buf->recycled) {
- bsize += cl->buf->last - cl->buf->pos;
- }
-
cl->next = NULL;
if (out) {
@@ -703,6 +682,7 @@ ngx_event_pipe_write_chain_to_temp_file(
{
ssize_t size, bsize;
ngx_buf_t *b;
+ ngx_uint_t prev_last_shadow;
ngx_chain_t *cl, *tl, *next, *out, **ll, **last_free, fl;
if (p->buf_to_file) {
@@ -719,6 +699,7 @@ ngx_event_pipe_write_chain_to_temp_file(
size = 0;
cl = out;
ll = NULL;
+ prev_last_shadow = 1;
ngx_log_debug1(NGX_LOG_DEBUG_EVENT, p->log, 0,
"pipe offset: %O", p->temp_file->offset);
@@ -726,16 +707,21 @@ ngx_event_pipe_write_chain_to_temp_file(
do {
bsize = cl->buf->last - cl->buf->pos;
- ngx_log_debug3(NGX_LOG_DEBUG_EVENT, p->log, 0,
- "pipe buf %p, pos %p, size: %z",
- cl->buf->start, cl->buf->pos, bsize);
+ ngx_log_debug4(NGX_LOG_DEBUG_EVENT, p->log, 0,
+ "pipe buf ls:%d %p, pos %p, size: %z",
+ cl->buf->last_shadow, cl->buf->start,
+ cl->buf->pos, bsize);
- if ((size + bsize > p->temp_file_write_size)
- || (p->temp_file->offset + size + bsize > p->max_temp_file_size))
+ if (prev_last_shadow
+ && ((size + bsize > p->temp_file_write_size)
+ || (p->temp_file->offset + size + bsize
+ > p->max_temp_file_size)))
{
break;
}
+ prev_last_shadow = cl->buf->last_shadow;
+
size += bsize;
ll = &cl->next;
cl = cl->next;
@@ -913,35 +899,6 @@ ngx_event_pipe_remove_shadow_links(ngx_b
}
-static ngx_inline void
-ngx_event_pipe_free_shadow_raw_buf(ngx_chain_t **free, ngx_buf_t *buf)
-{
- ngx_buf_t *s;
- ngx_chain_t *cl, **ll;
-
- if (buf->shadow == NULL) {
- return;
- }
-
- for (s = buf->shadow; !s->last_shadow; s = s->shadow) { /* void */ }
-
- ll = free;
-
- for (cl = *free; cl; cl = cl->next) {
- if (cl->buf == s) {
- *ll = cl->next;
- break;
- }
-
- if (cl->buf->shadow) {
- break;
- }
-
- ll = &cl->next;
- }
-}
-
-
ngx_int_t
ngx_event_pipe_add_free_buf(ngx_event_pipe_t *p, ngx_buf_t *b)
{
More information about the nginx
mailing list