QUIC: refactored ngx_quic_order_bufs() and ngx_quic_split_bufs().

They are replaced with ngx_quic_write_chain() and ngx_quic_read_chain().
These functions represent the API to data buffering.

The first function adds data of given size at given offset to the buffer.
Now it returns the unwritten part of the chain similar to c->send_chain().

The second function returns data of given size from the beginning of the buffer.
Its second argument and return value are swapped compared to
ngx_quic_split_bufs() to better match ngx_quic_write_chain().

Added, returned and stored data are regular ngx_chain_t/ngx_buf_t chains.
Missing data is marked with b->sync flag.

The functions are now used in both send and recv data chains in QUIC streams.

4 files changed, 117 insertions, 96 deletions

diff --git a/src/event/quic/ngx_event_quic_frames.c b/src/event/quic/ngx_event_quic_frames.c
index 71ed981e6..fe6d98043 100644
--- a/src/event/quic/ngx_event_quic_frames.c
+++ b/src/event/quic/ngx_event_quic_frames.c
@@ -13,10 +13,6 @@
 #define NGX_QUIC_BUFFER_SIZE  4096
 
 
-static ngx_chain_t *ngx_quic_split_bufs(ngx_connection_t *c, ngx_chain_t *in,
-    size_t len);
-
-
 ngx_quic_frame_t *
 ngx_quic_alloc_frame(ngx_connection_t *c)
 {
@@ -243,8 +239,8 @@ ngx_quic_split_frame(ngx_connection_t *c, ngx_quic_frame_t *f, size_t len)
     onf->length = shrink;
     nf->len = ngx_quic_create_frame(NULL, nf);
 
-    nf->data = ngx_quic_split_bufs(c, f->data, of->length);
-    if (nf->data == NGX_CHAIN_ERROR) {
+    f->data = ngx_quic_read_chain(c, &nf->data, of->length);
+    if (f->data == NGX_CHAIN_ERROR) {
         return NGX_ERROR;
     }
 
@@ -254,37 +250,48 @@ ngx_quic_split_frame(ngx_connection_t *c, ngx_quic_frame_t *f, size_t len)
 }
 
 
-static ngx_chain_t *
-ngx_quic_split_bufs(ngx_connection_t *c, ngx_chain_t *in, size_t len)
+ngx_chain_t *
+ngx_quic_read_chain(ngx_connection_t *c, ngx_chain_t **chain, off_t limit)
 {
-    size_t                  n;
+    off_t                   n;
     ngx_buf_t              *b;
-    ngx_chain_t            *out;
+    ngx_chain_t            *out, *in, *cl, **ll;
     ngx_quic_connection_t  *qc;
 
     qc = ngx_quic_get_connection(c);
 
-    while (in) {
-        n = ngx_buf_size(in->buf);
+    out = *chain;
+
+    for (ll = &out; *ll; ll = &(*ll)->next) {
+        b = (*ll)->buf;
 
-        if (n == len) {
-            out = in->next;
-            in->next = NULL;
-            return out;
+        if (b->sync) {
+            /* hole */
+            break;
         }
 
-        if (n > len) {
+        if (limit == 0) {
             break;
         }
 
-        len -= n;
-        in = in->next;
-    }
+        n = b->last - b->pos;
 
-    if (in == NULL) {
-        return NULL;
+        if (n > limit) {
+            goto split;
+        }
+
+        limit -= n;
     }
 
+    *chain = *ll;
+    *ll = NULL;
+
+    return out;
+
+split:
+
+    in = *ll;
+
     /* split in->buf by creating shadow bufs which reference it */
 
     if (in->buf->shadow == NULL) {
@@ -305,8 +312,8 @@ ngx_quic_split_bufs(ngx_connection_t *c, ngx_chain_t *in, size_t len)
         in->buf = b;
     }
 
-    out = ngx_alloc_chain_link(c->pool);
-    if (out == NULL) {
+    cl = ngx_alloc_chain_link(c->pool);
+    if (cl == NULL) {
         return NGX_CHAIN_ERROR;
     }
 
@@ -317,21 +324,23 @@ ngx_quic_split_bufs(ngx_connection_t *c, ngx_chain_t *in, size_t len)
     } else {
         b = ngx_alloc_buf(c->pool);
         if (b == NULL) {
-            ngx_free_chain(c->pool, out);
+            ngx_free_chain(c->pool, cl);
             return NGX_CHAIN_ERROR;
         }
     }
 
-    out->buf = b;
-    out->next = in->next;
+    cl->buf = b;
+    cl->next = in->next;
     in->next = NULL;
+    *chain = cl;
 
     *b = *in->buf;
     b->last_shadow = 0;
-    b->pos = b->pos + len;
+    b->pos += limit;
 
     in->buf->shadow = b;
-    in->buf->last = in->buf->pos + len;
+    in->buf->last = b->pos;
+    in->buf->last_buf = 0;
 
     return out;
 }
@@ -480,8 +489,8 @@ done:
 }
 
 
-ngx_int_t
-ngx_quic_order_bufs(ngx_connection_t *c, ngx_chain_t **out, ngx_chain_t *in,
+ngx_chain_t *
+ngx_quic_write_chain(ngx_connection_t *c, ngx_chain_t **chain, ngx_chain_t *in,
     off_t limit, off_t offset)
 {
     off_t         n;
@@ -490,18 +499,18 @@ ngx_quic_order_bufs(ngx_connection_t *c, ngx_chain_t **out, ngx_chain_t *in,
     ngx_chain_t  *cl, *sl;
 
     while (in && limit) {
-        cl = *out;
+        cl = *chain;
 
         if (cl == NULL) {
             cl = ngx_quic_alloc_buf(c);
             if (cl == NULL) {
-                return NGX_ERROR;
+                return NGX_CHAIN_ERROR;
             }
 
             cl->buf->last = cl->buf->end;
             cl->buf->sync = 1; /* hole */
             cl->next = NULL;
-            *out = cl;
+            *chain = cl;
         }
 
         b = cl->buf;
@@ -509,17 +518,25 @@ ngx_quic_order_bufs(ngx_connection_t *c, ngx_chain_t **out, ngx_chain_t *in,
 
         if (n <= offset) {
             offset -= n;
-            out = &cl->next;
+            chain = &cl->next;
             continue;
         }
 
         if (b->sync && offset > 0) {
-            sl = ngx_quic_split_bufs(c, cl, offset);
-            if (sl == NGX_CHAIN_ERROR) {
-                return NGX_ERROR;
+            /* split hole at offset */
+
+            b->sync = 0;
+
+            sl = ngx_quic_read_chain(c, &cl, offset);
+            if (cl == NGX_CHAIN_ERROR) {
+                return NGX_CHAIN_ERROR;
             }
 
-            cl->next = sl;
+            sl->buf->sync = 1;
+            cl->buf->sync = 1;
+
+            *chain = sl;
+            sl->next = cl;
             continue;
         }
 
@@ -541,18 +558,29 @@ ngx_quic_order_bufs(ngx_connection_t *c, ngx_chain_t **out, ngx_chain_t *in,
             }
         }
 
+        if (b->sync && p == b->last) {
+            b->sync = 0;
+            continue;
+        }
+
         if (b->sync && p != b->pos) {
-            sl = ngx_quic_split_bufs(c, cl, p - b->pos);
+            /* split hole at p - b->pos */
+
+            b->sync = 0;
+
+            sl = ngx_quic_read_chain(c, &cl, p - b->pos);
             if (sl == NGX_CHAIN_ERROR) {
-                return NGX_ERROR;
+                return NGX_CHAIN_ERROR;
             }
 
-            cl->next = sl;
-            cl->buf->sync = 0;
+            cl->buf->sync = 1;
+
+            *chain = sl;
+            sl->next = cl;
         }
     }
 
-    return NGX_OK;
+    return in;
 }
 
 
diff --git a/src/event/quic/ngx_event_quic_frames.h b/src/event/quic/ngx_event_quic_frames.h
index 1310f6dfd..19a5c1668 100644
--- a/src/event/quic/ngx_event_quic_frames.h
+++ b/src/event/quic/ngx_event_quic_frames.h
@@ -30,7 +30,9 @@ ngx_chain_t *ngx_quic_copy_chain(ngx_connection_t *c, ngx_chain_t *in,
     size_t limit);
 void ngx_quic_trim_bufs(ngx_chain_t *in, size_t size);
 void ngx_quic_free_bufs(ngx_connection_t *c, ngx_chain_t *in);
-ngx_int_t ngx_quic_order_bufs(ngx_connection_t *c, ngx_chain_t **out,
+ngx_chain_t *ngx_quic_read_chain(ngx_connection_t *c, ngx_chain_t **chain,
+    off_t limit);
+ngx_chain_t *ngx_quic_write_chain(ngx_connection_t *c, ngx_chain_t **chain,
     ngx_chain_t *in, off_t limit, off_t offset);
 
 #if (NGX_DEBUG)
diff --git a/src/event/quic/ngx_event_quic_ssl.c b/src/event/quic/ngx_event_quic_ssl.c
index d2f1237a5..dc87db41c 100644
--- a/src/event/quic/ngx_event_quic_ssl.c
+++ b/src/event/quic/ngx_event_quic_ssl.c
@@ -369,8 +369,14 @@ ngx_quic_handle_crypto_frame(ngx_connection_t *c, ngx_quic_header_t *pkt,
     }
 
     if (f->offset > ctx->crypto_received) {
-        return ngx_quic_order_bufs(c, &ctx->crypto, frame->data, f->length,
-                                   f->offset - ctx->crypto_received);
+        if (ngx_quic_write_chain(c, &ctx->crypto, frame->data, f->length,
+                                 f->offset - ctx->crypto_received)
+            == NGX_CHAIN_ERROR)
+        {
+            return NGX_ERROR;
+        }
+
+        return NGX_OK;
     }
 
     ngx_quic_trim_bufs(frame->data, ctx->crypto_received - f->offset);
diff --git a/src/event/quic/ngx_event_quic_streams.c b/src/event/quic/ngx_event_quic_streams.c
index d5facc270..d2984a345 100644
--- a/src/event/quic/ngx_event_quic_streams.c
+++ b/src/event/quic/ngx_event_quic_streams.c
@@ -732,9 +732,9 @@ ngx_quic_empty_handler(ngx_event_t *ev)
 static ssize_t
 ngx_quic_stream_recv(ngx_connection_t *c, u_char *buf, size_t size)
 {
-    ssize_t             len, n;
+    ssize_t             len;
     ngx_buf_t          *b;
-    ngx_chain_t        *cl, **ll;
+    ngx_chain_t        *cl, *in;
     ngx_event_t        *rev;
     ngx_connection_t   *pc;
     ngx_quic_stream_t  *qs;
@@ -764,33 +764,20 @@ ngx_quic_stream_recv(ngx_connection_t *c, u_char *buf, size_t size)
         return NGX_AGAIN;
     }
 
-    len = 0;
-    cl = qs->in;
-
-    for (ll = &cl; *ll; ll = &(*ll)->next) {
-        b = (*ll)->buf;
-
-        if (b->sync) {
-            /* hole */
-            break;
-        }
-
-        n = ngx_min(b->last - b->pos, (ssize_t) size);
-        buf = ngx_cpymem(buf, b->pos, n);
+    in = ngx_quic_read_chain(pc, &qs->in, size);
+    if (in == NGX_CHAIN_ERROR) {
+        return NGX_ERROR;
+    }
 
-        len += n;
-        size -= n;
-        b->pos += n;
+    len = 0;
 
-        if (b->pos != b->last) {
-            break;
-        }
+    for (cl = in; cl; cl = cl->next) {
+        b = cl->buf;
+        len += b->last - b->pos;
+        buf = ngx_cpymem(buf, b->pos, b->last - b->pos);
     }
 
-    qs->in = *ll;
-    *ll = NULL;
-
-    ngx_quic_free_bufs(pc, cl);
+    ngx_quic_free_bufs(pc, in);
 
     if (qs->in == NULL) {
         rev->ready = rev->pending_eof;
@@ -837,10 +824,9 @@ ngx_quic_stream_send(ngx_connection_t *c, u_char *buf, size_t size)
 static ngx_chain_t *
 ngx_quic_stream_send_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
 {
-    size_t                  n, flow;
-    ngx_buf_t              *b;
+    off_t                   n, flow;
     ngx_event_t            *wev;
-    ngx_chain_t            *out, **ll;
+    ngx_chain_t            *out, *cl;
     ngx_connection_t       *pc;
     ngx_quic_frame_t       *frame;
     ngx_quic_stream_t      *qs;
@@ -861,38 +847,35 @@ ngx_quic_stream_send_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
         return in;
     }
 
-    n = (limit && (size_t) limit < flow) ? (size_t) limit : flow;
-
-    if (ngx_quic_order_bufs(pc, &qs->out, in, n, 0) != NGX_OK) {
-        return NGX_CHAIN_ERROR;
+    if (limit == 0 || limit > flow) {
+        limit = flow;
     }
 
     n = 0;
-    out = qs->out;
-
-    for (ll = &out; *ll; ll = &(*ll)->next) {
-        b = (*ll)->buf;
 
-        if (b->sync) {
-            /* hole */
+    for (cl = in; cl; cl = cl->next) {
+        n += cl->buf->last - cl->buf->pos;
+        if (n >= limit) {
+            n = limit;
             break;
         }
+    }
 
-        n += b->last - b->pos;
+    in = ngx_quic_write_chain(pc, &qs->out, in, n, 0);
+    if (in == NGX_CHAIN_ERROR) {
+        return NGX_CHAIN_ERROR;
     }
 
-    qs->out = *ll;
-    *ll = NULL;
+    out = ngx_quic_read_chain(pc, &qs->out, n);
+    if (out == NGX_CHAIN_ERROR) {
+        return NGX_CHAIN_ERROR;
+    }
 
     frame = ngx_quic_alloc_frame(pc);
     if (frame == NULL) {
         return NGX_CHAIN_ERROR;
     }
 
-    while (in && ngx_buf_size(in->buf) == 0) {
-        in = in->next;
-    }
-
     frame->level = ssl_encryption_application;
     frame->type = NGX_QUIC_FT_STREAM;
     frame->data = out;
@@ -909,7 +892,9 @@ ngx_quic_stream_send_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
 
     ngx_quic_queue_frame(qc, frame);
 
-    wev->ready = (n < flow) ? 1 : 0;
+    if (in) {
+        wev->ready = 0;
+    }
 
     ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
                    "quic send_chain sent:%uz", n);
@@ -1113,9 +1098,9 @@ ngx_quic_handle_stream_frame(ngx_connection_t *c, ngx_quic_header_t *pkt,
         qs->final_size = last;
     }
 
-    if (ngx_quic_order_bufs(c, &qs->in, frame->data, f->length,
-                            f->offset - qs->recv_offset)
-        != NGX_OK)
+    if (ngx_quic_write_chain(c, &qs->in, frame->data, f->length,
+                             f->offset - qs->recv_offset)
+        == NGX_CHAIN_ERROR)
     {
         return NGX_ERROR;
     }