proxy_store corrupted/cut flv files

[Igor Sysoev]

On Sun, Dec 23, 2007 at 09:49:47PM +0200, Dmitry Sherman wrote:

> I've tried to use the proxy_store feature to cache flv files on the nginx
> server,
> 
> For some reason most of the files are corrupted, when I compared the
> original files to the cached I found the cached corrupted files are little
> smaller (some of the more and some less smaller) then the original.
> 
>  
> 
> The connection between the servers is fast and there is no congestion on the
> line.
> 
> I tried to upgrade to nginx 0.6 (latest) - didn't help.
> 
>  
> 
> When I cache small images everything goes well. The FLV files size varies
> from 1mb to 20mb , the small (1mb) files also have the same problem.
> 
>  
> 
> At the end, the player cannot play those files at all, what I do now is
> running rsync between the servers, it solves me the problem but actually its
> not a good solution because I need new added flv files to work instantly
> without waiting for the rsync schedule.

nginx stores response if it has 200 status and it has been received without
errors. However, nginx did not test a "Content-Length" header and real data
length. The attached patch fixes this.

Although, flv will be stored incorrectly, if it was requested not from
their beginning (?start=XXX).


-- 
Igor Sysoev
http://sysoev.ru/en/
-------------- next part --------------
Index: src/http/ngx_http_upstream.c
===================================================================
--- src/http/ngx_http_upstream.c	(revision 1097)
+++ src/http/ngx_http_upstream.c	(working copy)
@@ -1352,6 +1352,8 @@
     r->headers_out.status = u->headers_in.status_n;
     r->headers_out.status_line = u->headers_in.status_line;
 
+    u->headers_in.content_length_n = r->headers_out.content_length_n;
+
     if (r->headers_out.content_length_n != -1) {
         u->length = (size_t) r->headers_out.content_length_n;
 
@@ -1955,6 +1957,7 @@
 static void
 ngx_http_upstream_process_body(ngx_event_t *ev)
 {
+    ngx_temp_file_t      *tf;
     ngx_event_pipe_t     *p;
     ngx_connection_t     *c, *downstream;
     ngx_http_log_ctx_t   *ctx;
@@ -2049,18 +2052,22 @@
 
         if (u->store) {
 
-            if (p->upstream_eof && u->headers_in.status_n == NGX_HTTP_OK) {
+            tf = u->pipe->temp_file;
 
+            if (p->upstream_eof
+                && u->headers_in.status_n == NGX_HTTP_OK
+                && (u->headers_in.content_length_n == -1
+                    || (u->headers_in.content_length_n == tf->offset)))
+            {
                 ngx_http_upstream_store(r, u);
 
             } else if ((p->upstream_error
                         || (p->upstream_eof
                             && u->headers_in.status_n != NGX_HTTP_OK))
-                       && u->pipe->temp_file->file.fd != NGX_INVALID_FILE)
+                       && tf->file.fd != NGX_INVALID_FILE)
             {
-                if (ngx_delete_file(u->pipe->temp_file->file.name.data)
-                    == NGX_FILE_ERROR)
-                {
+                if (ngx_delete_file(tf->file.name.data) == NGX_FILE_ERROR) {
+
                     ngx_log_error(NGX_LOG_CRIT, r->connection->log, ngx_errno,
                                   ngx_delete_file_n " \"%s\" failed",
                                   u->pipe->temp_file->file.name.data);
Index: src/http/ngx_http_upstream.h
===================================================================
--- src/http/ngx_http_upstream.h	(revision 1097)
+++ src/http/ngx_http_upstream.h	(working copy)
@@ -192,6 +192,8 @@
     ngx_table_elt_t                *content_encoding;
 #endif
 
+    off_t                           content_length_n;
+
     ngx_array_t                     cache_control;
 } ngx_http_upstream_headers_in_t;