Upstream: re-resolvable servers.

Specifying the upstream server by a hostname together with the
"resolve" parameter will make the hostname to be periodically
resolved, and upstream servers added/removed as necessary.

This requires a "resolver" at the "http" configuration block.

The "resolver_timeout" parameter also affects when the failed
DNS requests will be attempted again.  Responses with NXDOMAIN
will be attempted again in 10 seconds.

Upstream has a configuration generation number that is incremented each
time servers are added/removed to the primary/backup list.  This number
is remembered by the peer.init method, and if peer.get detects a change
in configuration, it returns NGX_BUSY.

Each server has a reference counter.  It is incremented by peer.get and
decremented by peer.free.  When a server is removed, it is removed from
the list of servers and is marked as "zombie".  The memory allocated by
a zombie peer is freed only when its reference count becomes zero.

Co-authored-by: Roman Arutyunyan <arut@nginx.com>
Co-authored-by: Sergey Kandaurov <pluknet@nginx.com>
Co-authored-by: Vladimir Homutov <vl@nginx.com>

1 files changed, 361 insertions, 1 deletions

diff --git a/src/stream/ngx_stream_upstream_zone_module.c b/src/stream/ngx_stream_upstream_zone_module.c
index 80d42fa00..d731baf31 100644
--- a/src/stream/ngx_stream_upstream_zone_module.c
+++ b/src/stream/ngx_stream_upstream_zone_module.c
@@ -18,6 +18,13 @@ static ngx_stream_upstream_rr_peers_t *ngx_stream_upstream_zone_copy_peers(
     ngx_slab_pool_t *shpool, ngx_stream_upstream_srv_conf_t *uscf);
 static ngx_stream_upstream_rr_peer_t *ngx_stream_upstream_zone_copy_peer(
     ngx_stream_upstream_rr_peers_t *peers, ngx_stream_upstream_rr_peer_t *src);
+static void ngx_stream_upstream_zone_set_single(
+    ngx_stream_upstream_srv_conf_t *uscf);
+static void ngx_stream_upstream_zone_remove_peer_locked(
+    ngx_stream_upstream_rr_peers_t *peers, ngx_stream_upstream_rr_peer_t *peer);
+static ngx_int_t ngx_stream_upstream_zone_init_worker(ngx_cycle_t *cycle);
+static void ngx_stream_upstream_zone_resolve_timer(ngx_event_t *event);
+static void ngx_stream_upstream_zone_resolve_handler(ngx_resolver_ctx_t *ctx);
 
 
 static ngx_command_t  ngx_stream_upstream_zone_commands[] = {
@@ -52,7 +59,7 @@ ngx_module_t  ngx_stream_upstream_zone_module = {
     NGX_STREAM_MODULE,                     /* module type */
     NULL,                                  /* init master */
     NULL,                                  /* init module */
-    NULL,                                  /* init process */
+    ngx_stream_upstream_zone_init_worker,  /* init process */
     NULL,                                  /* init thread */
     NULL,                                  /* exit thread */
     NULL,                                  /* exit process */
@@ -185,9 +192,15 @@ ngx_stream_upstream_zone_copy_peers(ngx_slab_pool_t *shpool,
     ngx_stream_upstream_srv_conf_t *uscf)
 {
     ngx_str_t                       *name;
+    ngx_uint_t                      *config;
     ngx_stream_upstream_rr_peer_t   *peer, **peerp;
     ngx_stream_upstream_rr_peers_t  *peers, *backup;
 
+    config = ngx_slab_calloc(shpool, sizeof(ngx_uint_t));
+    if (config == NULL) {
+        return NULL;
+    }
+
     peers = ngx_slab_alloc(shpool, sizeof(ngx_stream_upstream_rr_peers_t));
     if (peers == NULL) {
         return NULL;
@@ -211,6 +224,7 @@ ngx_stream_upstream_zone_copy_peers(ngx_slab_pool_t *shpool,
     peers->name = name;
 
     peers->shpool = shpool;
+    peers->config = config;
 
     for (peerp = &peers->peer; *peerp; peerp = &peer->next) {
         /* pool is unlocked */
@@ -220,6 +234,17 @@ ngx_stream_upstream_zone_copy_peers(ngx_slab_pool_t *shpool,
         }
 
         *peerp = peer;
+        (*peers->config)++;
+    }
+
+    for (peerp = &peers->resolve; *peerp; peerp = &peer->next) {
+        peer = ngx_stream_upstream_zone_copy_peer(peers, *peerp);
+        if (peer == NULL) {
+            return NULL;
+        }
+
+        *peerp = peer;
+        (*peers->config)++;
     }
 
     if (peers->next == NULL) {
@@ -236,6 +261,7 @@ ngx_stream_upstream_zone_copy_peers(ngx_slab_pool_t *shpool,
     backup->name = name;
 
     backup->shpool = shpool;
+    backup->config = config;
 
     for (peerp = &backup->peer; *peerp; peerp = &peer->next) {
         /* pool is unlocked */
@@ -245,6 +271,17 @@ ngx_stream_upstream_zone_copy_peers(ngx_slab_pool_t *shpool,
         }
 
         *peerp = peer;
+        (*backup->config)++;
+    }
+
+    for (peerp = &backup->resolve; *peerp; peerp = &peer->next) {
+        peer = ngx_stream_upstream_zone_copy_peer(backup, *peerp);
+        if (peer == NULL) {
+            return NULL;
+        }
+
+        *peerp = peer;
+        (*backup->config)++;
     }
 
     peers->next = backup;
@@ -276,6 +313,7 @@ ngx_stream_upstream_zone_copy_peer(ngx_stream_upstream_rr_peers_t *peers,
         dst->sockaddr = NULL;
         dst->name.data = NULL;
         dst->server.data = NULL;
+        dst->host = NULL;
     }
 
     dst->sockaddr = ngx_slab_calloc_locked(pool, sizeof(ngx_sockaddr_t));
@@ -298,12 +336,37 @@ ngx_stream_upstream_zone_copy_peer(ngx_stream_upstream_rr_peers_t *peers,
         }
 
         ngx_memcpy(dst->server.data, src->server.data, src->server.len);
+
+        if (src->host) {
+            dst->host = ngx_slab_calloc_locked(pool,
+                                            sizeof(ngx_stream_upstream_host_t));
+            if (dst->host == NULL) {
+                goto failed;
+            }
+
+            dst->host->name.data = ngx_slab_alloc_locked(pool,
+                                                         src->host->name.len);
+            if (dst->host->name.data == NULL) {
+                goto failed;
+            }
+
+            dst->host->peers = peers;
+            dst->host->peer = dst;
+
+            dst->host->name.len = src->host->name.len;
+            ngx_memcpy(dst->host->name.data, src->host->name.data,
+                       src->host->name.len);
+        }
     }
 
     return dst;
 
 failed:
 
+    if (dst->host) {
+        ngx_slab_free_locked(pool, dst->host);
+    }
+
     if (dst->server.data) {
         ngx_slab_free_locked(pool, dst->server.data);
     }
@@ -320,3 +383,300 @@ failed:
 
     return NULL;
 }
+
+
+static void
+ngx_stream_upstream_zone_set_single(ngx_stream_upstream_srv_conf_t *uscf)
+{
+    ngx_stream_upstream_rr_peers_t  *peers;
+
+    peers = uscf->peer.data;
+
+    if (peers->number == 1
+        && (peers->next == NULL || peers->next->number == 0))
+    {
+        peers->single = 1;
+
+    } else {
+        peers->single = 0;
+    }
+}
+
+
+static void
+ngx_stream_upstream_zone_remove_peer_locked(
+    ngx_stream_upstream_rr_peers_t *peers, ngx_stream_upstream_rr_peer_t *peer)
+{
+    peers->total_weight -= peer->weight;
+    peers->number--;
+    peers->tries -= (peer->down == 0);
+    (*peers->config)++;
+    peers->weighted = (peers->total_weight != peers->number);
+
+    ngx_stream_upstream_rr_peer_free(peers, peer);
+}
+
+
+static ngx_int_t
+ngx_stream_upstream_zone_init_worker(ngx_cycle_t *cycle)
+{
+    ngx_uint_t                        i;
+    ngx_event_t                      *event;
+    ngx_stream_upstream_rr_peer_t    *peer;
+    ngx_stream_upstream_rr_peers_t   *peers;
+    ngx_stream_upstream_srv_conf_t   *uscf, **uscfp;
+    ngx_stream_upstream_main_conf_t  *umcf;
+
+    if (ngx_process != NGX_PROCESS_WORKER
+        && ngx_process != NGX_PROCESS_SINGLE)
+    {
+        return NGX_OK;
+    }
+
+    umcf = ngx_stream_cycle_get_module_main_conf(cycle,
+                                                 ngx_stream_upstream_module);
+
+    if (umcf == NULL) {
+        return NGX_OK;
+    }
+
+    uscfp = umcf->upstreams.elts;
+
+    for (i = 0; i < umcf->upstreams.nelts; i++) {
+
+        uscf = uscfp[i];
+
+        if (uscf->shm_zone == NULL) {
+            continue;
+        }
+
+        peers = uscf->peer.data;
+
+        do {
+            ngx_stream_upstream_rr_peers_wlock(peers);
+
+            for (peer = peers->resolve; peer; peer = peer->next) {
+
+                if (peer->host->worker != ngx_worker) {
+                    continue;
+                }
+
+                event = &peer->host->event;
+                ngx_memzero(event, sizeof(ngx_event_t));
+
+                event->data = uscf;
+                event->handler = ngx_stream_upstream_zone_resolve_timer;
+                event->log = cycle->log;
+                event->cancelable = 1;
+
+                ngx_add_timer(event, 1);
+            }
+
+            ngx_stream_upstream_rr_peers_unlock(peers);
+
+            peers = peers->next;
+
+        } while (peers);
+    }
+
+    return NGX_OK;
+}
+
+
+static void
+ngx_stream_upstream_zone_resolve_timer(ngx_event_t *event)
+{
+    ngx_resolver_ctx_t              *ctx;
+    ngx_stream_upstream_host_t      *host;
+    ngx_stream_upstream_srv_conf_t  *uscf;
+
+    host = (ngx_stream_upstream_host_t *) event;
+    uscf = event->data;
+
+    ctx = ngx_resolve_start(uscf->resolver, NULL);
+    if (ctx == NULL) {
+        goto retry;
+    }
+
+    if (ctx == NGX_NO_RESOLVER) {
+        ngx_log_error(NGX_LOG_ERR, event->log, 0,
+                      "no resolver defined to resolve %V", &host->name);
+        return;
+    }
+
+    ctx->name = host->name;
+    ctx->handler = ngx_stream_upstream_zone_resolve_handler;
+    ctx->data = host;
+    ctx->timeout = uscf->resolver_timeout;
+    ctx->cancelable = 1;
+
+    if (ngx_resolve_name(ctx) == NGX_OK) {
+        return;
+    }
+
+retry:
+
+    ngx_add_timer(event, ngx_max(uscf->resolver_timeout, 1000));
+}
+
+
+static void
+ngx_stream_upstream_zone_resolve_handler(ngx_resolver_ctx_t *ctx)
+{
+    time_t                           now;
+    in_port_t                        port;
+    ngx_msec_t                       timer;
+    ngx_uint_t                       i, j;
+    ngx_event_t                     *event;
+    ngx_resolver_addr_t             *addr;
+    ngx_stream_upstream_host_t      *host;
+    ngx_stream_upstream_rr_peer_t   *peer, *template, **peerp;
+    ngx_stream_upstream_rr_peers_t  *peers;
+    ngx_stream_upstream_srv_conf_t  *uscf;
+
+    host = ctx->data;
+    event = &host->event;
+    uscf = event->data;
+    peers = host->peers;
+    template = host->peer;
+
+    ngx_stream_upstream_rr_peers_wlock(peers);
+
+    now = ngx_time();
+
+    if (ctx->state) {
+        ngx_log_error(NGX_LOG_ERR, event->log, 0,
+                      "%V could not be resolved (%i: %s)",
+                      &ctx->name, ctx->state,
+                      ngx_resolver_strerror(ctx->state));
+
+        if (ctx->state != NGX_RESOLVE_NXDOMAIN) {
+            ngx_stream_upstream_rr_peers_unlock(peers);
+
+            ngx_resolve_name_done(ctx);
+
+            ngx_add_timer(event, ngx_max(uscf->resolver_timeout, 1000));
+            return;
+        }
+
+        /* NGX_RESOLVE_NXDOMAIN */
+
+        ctx->naddrs = 0;
+    }
+
+#if (NGX_DEBUG)
+    {
+    u_char  text[NGX_SOCKADDR_STRLEN];
+    size_t  len;
+
+    for (i = 0; i < ctx->naddrs; i++) {
+        len = ngx_sock_ntop(ctx->addrs[i].sockaddr, ctx->addrs[i].socklen,
+                            text, NGX_SOCKADDR_STRLEN, 0);
+
+        ngx_log_debug3(NGX_LOG_DEBUG_STREAM, event->log, 0,
+                       "name %V was resolved to %*s", &host->name, len, text);
+    }
+    }
+#endif
+
+    for (peerp = &peers->peer; *peerp; /* void */ ) {
+        peer = *peerp;
+
+        if (peer->host != host) {
+            goto next;
+        }
+
+        for (j = 0; j < ctx->naddrs; j++) {
+
+            addr = &ctx->addrs[j];
+
+            if (addr->name.len == 0
+                && ngx_cmp_sockaddr(peer->sockaddr, peer->socklen,
+                                    addr->sockaddr, addr->socklen, 0)
+                   == NGX_OK)
+            {
+                addr->name.len = 1;
+                goto next;
+            }
+        }
+
+        *peerp = peer->next;
+        ngx_stream_upstream_zone_remove_peer_locked(peers, peer);
+
+        ngx_stream_upstream_zone_set_single(uscf);
+
+        continue;
+
+    next:
+
+        peerp = &peer->next;
+    }
+
+    for (i = 0; i < ctx->naddrs; i++) {
+
+        addr = &ctx->addrs[i];
+
+        if (addr->name.len == 1) {
+            addr->name.len = 0;
+            continue;
+        }
+
+        ngx_shmtx_lock(&peers->shpool->mutex);
+        peer = ngx_stream_upstream_zone_copy_peer(peers, NULL);
+        ngx_shmtx_unlock(&peers->shpool->mutex);
+        if (peer == NULL) {
+            ngx_log_error(NGX_LOG_ERR, event->log, 0,
+                          "cannot add new server to upstream \"%V\", "
+                          "memory exhausted", peers->name);
+            break;
+        }
+
+        ngx_memcpy(peer->sockaddr, addr->sockaddr, addr->socklen);
+
+        port = ((struct sockaddr_in *) template->sockaddr)->sin_port;
+
+        switch (peer->sockaddr->sa_family) {
+#if (NGX_HAVE_INET6)
+        case AF_INET6:
+            ((struct sockaddr_in6 *) peer->sockaddr)->sin6_port = port;
+            break;
+#endif
+        default: /* AF_INET */
+            ((struct sockaddr_in *) peer->sockaddr)->sin_port = port;
+        }
+
+        peer->socklen = addr->socklen;
+
+        peer->name.len = ngx_sock_ntop(peer->sockaddr, peer->socklen,
+                                       peer->name.data, NGX_SOCKADDR_STRLEN, 1);
+
+        peer->host = template->host;
+        peer->server = template->server;
+
+        peer->weight = template->weight;
+        peer->effective_weight = peer->weight;
+        peer->max_conns = template->max_conns;
+        peer->max_fails = template->max_fails;
+        peer->fail_timeout = template->fail_timeout;
+        peer->down = template->down;
+
+        *peerp = peer;
+        peerp = &peer->next;
+
+        peers->number++;
+        peers->tries += (peer->down == 0);
+        peers->total_weight += peer->weight;
+        peers->weighted = (peers->total_weight != peers->number);
+        (*peers->config)++;
+
+        ngx_stream_upstream_zone_set_single(uscf);
+    }
+
+    ngx_stream_upstream_rr_peers_unlock(peers);
+
+    timer = (ngx_msec_t) 1000 * (ctx->valid > now ? ctx->valid - now + 1 : 1);
+
+    ngx_resolve_name_done(ctx);
+
+    ngx_add_timer(event, timer);
+}