MEDIUM: server: implement "purge server" CLI

Define and implement a new CLI command "purge server". This is similar
to "del server", except that deletion is performed in background if it
cannot be completed due to a transient state.

To schedule background deletion, server instance is flagged with
SRV_F_TO_PURGE and inserted into parent proxy <servers_purge> list.
Finally, proxy purge task is scheduled. This will periodically call
srv_check_for_deletion() : on success the server instance is finally
freed (unless its refcount is non null, in which case server free should
occur soon after).

A programmed background server deletion can be cancelled via
srv_cancel_purge() : this simply remove SRV_F_TO_PURGE. This action is
performed when the server state is changed (even to remain in
maintenance).

After purge cancellation, the server will be removed from the parent
proxy list via the task handler. This is not performed directly when
removing the server flag as this may simplify future evolutions, in
particular if thread isolation is reduced and proxy lock becomes
required to manipulate purge list.

diff --git a/doc/management.txt b/doc/management.txt
index 5d0a7322dbf4597759e88c95e6e9af84b3ed51cf..3e2cca5c830aad6a4386edb2b9d14f885fce8388 100644 (file)
--- a/doc/management.txt
+++ b/doc/management.txt
@@ -2597,6 +2597,13 @@ publish backend <backend>
   operation of "unpublish backend" command. This command is restricted and can
   only be issued on sockets configured for levels "operator" or "admin".
 
+purge server <backend>/<server>
+  This is an alternative command to "del server". Its main difference is that
+  if the server cannot be immediately removed due to remaining active or idle
+  connections, the server is flagged for background deletion instead of
+  rejecting the command. A scheduled deletion is cancelled if the server state
+  is explicitly reset, even if still in maintenance mode.
+
 quit
   Close the connection when in interactive mode.
 

diff --git a/include/haproxy/server-t.h b/include/haproxy/server-t.h
index 24ed276378e52a01f93d3bbc18541a062d197871..0c6eda227966be934a04279c048248c9b5959712 100644 (file)
--- a/include/haproxy/server-t.h
+++ b/include/haproxy/server-t.h
@@ -176,6 +176,7 @@ enum srv_init_state {
 #define SRV_F_UDP_GSO_NOTSUPP 0x40000    /* UDP GSO is disabled due to a previous error encountered */
 #define SRV_F_NAME_REFD    0x80000       /* this server's name is statically referenced (use-server, track, sample arg) */
 #define SRV_F_UMODIFIED    0x100000      /* at least one setting has been explicitely set via a server keyword */
+#define SRV_F_TO_PURGE     0x200000      /* server will be deleted when unused */
 
 /* configured server options for send-proxy (server->pp_opts) */
 #define SRV_PP_V1               0x0001   /* proxy protocol version 1 */
@@ -354,6 +355,7 @@ struct server {
        struct list el_px;                      /* attach point in parent proxy */
        struct list el_alt;                     /* extra attach point for several usages
                                                 * - for <servers_to_init> global list during parsing
+                                                * - for parent proxy purge list at runtime
                                                 */
        int cklen;                              /* the len of the cookie, to speed up checks */
        int rdr_len;                            /* the length of the redirection prefix */

diff --git a/include/haproxy/server.h b/include/haproxy/server.h
index e5e7f2d9621c640c4b58aebaee76f56804b93c44..0f80a946755e0d3743ca07741361bd10eaefea15 100644 (file)
--- a/include/haproxy/server.h
+++ b/include/haproxy/server.h
@@ -30,6 +30,7 @@
 #include <haproxy/applet-t.h>
 #include <haproxy/arg-t.h>
 #include <haproxy/counters.h>
+#include <haproxy/errors.h>
 #include <haproxy/freq_ctr.h>
 #include <haproxy/proxy-t.h>
 #include <haproxy/resolvers-t.h>
@@ -265,6 +266,18 @@ static inline int server_is_draining(const struct server *s)
        return !s->uweight || (s->cur_admin & SRV_ADMF_DRAIN);
 }
 
+/* Cancel the deletion of <s> server. */
+static inline void srv_cancel_purge(struct server *s)
+{
+       /* Only server flag is reset, Cannot remove it from proxy purge list
+        * without lock. This is performed instead during purge task execution.
+        */
+       if (s->flags & SRV_F_TO_PURGE) {
+               ha_notice("Server '%s/%s' purge cancelled.\n", s->proxy->id, s->id);
+               s->flags &= ~SRV_F_TO_PURGE;
+       }
+}
+
 /* Puts server <s> into maintenance mode, and propagate that status down to all
  * tracking servers.
  */
@@ -272,6 +285,7 @@ static inline void srv_adm_set_maint(struct server *s)
 {
        srv_set_admin_flag(s, SRV_ADMF_FMAINT, SRV_ADM_STCHGC_NONE);
        srv_clr_admin_flag(s, SRV_ADMF_FDRAIN);
+       srv_cancel_purge(s);
 }
 
 /* Puts server <s> into drain mode, and propagate that status down to all
@@ -281,6 +295,7 @@ static inline void srv_adm_set_drain(struct server *s)
 {
        srv_set_admin_flag(s, SRV_ADMF_FDRAIN, SRV_ADM_STCHGC_NONE);
        srv_clr_admin_flag(s, SRV_ADMF_FMAINT);
+       srv_cancel_purge(s);
 }
 
 /* Puts server <s> into ready mode, and propagate that status down to all
@@ -290,6 +305,7 @@ static inline void srv_adm_set_ready(struct server *s)
 {
        srv_clr_admin_flag(s, SRV_ADMF_FDRAIN);
        srv_clr_admin_flag(s, SRV_ADMF_FMAINT);
+       srv_cancel_purge(s);
 }
 
 static inline void srv_set_init_state(struct server *srv)
@@ -407,11 +423,7 @@ static inline void srv_detach(struct server *srv)
        struct proxy *px = srv->proxy;
 
        LIST_DEL_INIT(&srv->el_px);
-
-       if (global.mode & MODE_STARTING) {
-               /* Necessary if startup is interrupted due to an early error before check_config_validity(). */
-               LIST_DEL_INIT(&srv->el_alt);
-       }
+       LIST_DEL_INIT(&srv->el_alt);
 
        /* Reset the proxy's ready_srv if it was this one. */
        HA_ATOMIC_CAS(&px->ready_srv, &srv, NULL);

diff --git a/src/proxy.c b/src/proxy.c
index 14ccd1edc8bb1e8619d1af8d84d6cf78cc24584f..0c8da9d0a0c9487df5202e02246128c7d02d15dc 100644 (file)
--- a/src/proxy.c
+++ b/src/proxy.c
@@ -510,6 +510,40 @@ void proxy_drop(struct proxy *p)
        ha_free(&p);
 }
 
+/* Try to delete servers flagged for purge in <px> backend. */
+static int proxy_purge_servers(struct proxy *px)
+{
+       struct server *srv;
+       int ret;
+
+       while (!LIST_ISEMPTY(&px->servers_purge)) {
+               srv = LIST_NEXT(&px->servers_purge, struct server *, el_alt);
+               if (!(srv->flags & SRV_F_TO_PURGE)) {
+                       LIST_DEL_INIT(&srv->el_alt);
+                       continue;
+               }
+
+               /* TODO unnecessary server lookup */
+               ret = srv_check_for_deletion(px->id, srv->id, PR_DEL_MODE_SYNC, NULL, NULL, NULL);
+               if (ret < 0) {
+                       srv_cancel_purge(srv);
+                       LIST_DEL_INIT(&srv->el_alt);
+               }
+               else if (ret > 0) {
+                       ha_notice("Server '%s/%s' purged.\n", px->id, srv->id);
+                       srv_unregister(srv);
+                       srv_drop(srv);
+                       /* TODO yield after nth deletion performed in a row */
+               }
+               else {
+                       /* Interrupt work on first server not yet removable. */
+                       break;
+               }
+       }
+
+       return 0;
+}
+
 /* Proxy purge task handler, only used by user configured backend instances.
  * Its objective is to remove any servers flagged with purge or the backend
  * instance itself.
@@ -518,6 +552,11 @@ struct task *proxy_process_purge(struct task *t, void *context, unsigned int sta
 {
        struct proxy *px = context;
 
+       /* Must not dereference <px> before entering thread isolation and
+        * checking task state, as proxy_drop() performed outside isolation
+        * during "del backend".
+        */
+
        thread_isolate_full();
 
        if (t->state & TASK_KILLED) {
@@ -527,6 +566,10 @@ struct task *proxy_process_purge(struct task *t, void *context, unsigned int sta
                goto requeue;
        }
 
+       if (!LIST_ISEMPTY(&px->servers_purge))
+               proxy_purge_servers(px);
+
+       /* Set next task execution or cancel it if no more work to do. */
        if (!LIST_ISEMPTY(&px->servers_purge))
                t->expire = tick_add(now_ms, MS_TO_TICKS(1000));
        else

diff --git a/src/server.c b/src/server.c
index bda1e856f955cfe6ae2ead3ceab2a7884449dd1d..e29b9e7a6762bbf0e8f839308bf2305b937cb3a2 100644 (file)
--- a/src/server.c
+++ b/src/server.c
@@ -3178,6 +3178,7 @@ struct server *new_server(struct proxy *proxy)
        srv->obj_type = OBJ_TYPE_SERVER;
        srv->proxy = proxy;
        LIST_APPEND(&proxy->servers, &srv->el_px);
+       LIST_INIT(&srv->el_alt);
        LIST_INIT(&srv->srv_rec_item);
        LIST_INIT(&srv->ip_rec_item);
        LIST_INIT(&srv->pp_tlvs);
@@ -7108,6 +7109,57 @@ out:
        return 1;
 }
 
+/* Parser for "purge server" CLI command. Always return 1. */
+static int cli_parse_purge_server(char **args, char *payload, struct appctx *appctx, void *private)
+{
+       struct proxy *be;
+       struct server *srv;
+       const char *be_name, *sv_name, *msg;
+       int ret;
+
+       if (!cli_has_level(appctx, ACCESS_LVL_ADMIN))
+               return 1;
+
+       ++args;
+
+       thread_isolate_full();
+
+       if (parse_be_srv(args[1], &be_name, &sv_name, &msg)) {
+               cli_err(appctx, msg);
+               goto out;
+       }
+
+       ret = srv_check_for_deletion(be_name, sv_name, &be, &srv, &msg);
+       if (ret < 0) {
+               /* Unremovable server, cancel purge. */
+               cli_err(appctx, msg);
+               goto out;
+       }
+
+       if (!ret) {
+               /* Cannot remove server yet, schedule purging. */
+               if (!LIST_INLIST(&srv->el_alt))
+                       LIST_APPEND(&be->servers_purge, &srv->el_alt);
+               srv->flags |= SRV_F_TO_PURGE;
+
+               if (!tick_isset(be->purge_task->expire))
+                       be->purge_task->expire = tick_add(now_ms, 1000);
+               task_queue(be->purge_task);
+               ha_notice("Server '%s/%s' flagged for deletion.\n", be->id, srv->id);
+               cli_msg(appctx, LOG_INFO, "Server scheduled for purge.\n");
+       }
+       else {
+               srv_unregister(srv);
+               ha_notice("Server %s/%s deleted.\n", be->id, srv->id);
+               cli_msg(appctx, LOG_INFO, "Server deleted.\n");
+               srv_drop(srv);
+       }
+
+ out:
+       thread_release();
+       return 1;
+}
+
 /* Reset the statistics counters of a single server, invoked from the
  * "clear counters server <backend>/<server> [force]" CLI command (dispatched
  * by cli_parse_clear_counters() in stats.c, since "clear counters" is a
@@ -7174,6 +7226,7 @@ static struct cli_kw_list cli_kws = {{ },{
        { { "set", "weight", NULL },             "set weight <bk>/<srv>  (DEPRECATED)     : change a server's weight (use 'set server' instead)",         cli_parse_set_weight },
        { { "add", "server", NULL },             "add server <bk>/<srv>                   : create a new server",                                         cli_parse_add_server, cli_io_handler_add_server },
        { { "del", "server", NULL },             "del server <bk>/<srv>                   : remove a server",                                             cli_parse_delete_server, NULL },
+       { { "purge", "server", NULL },           "purge server <bk>/<srv>                 : remove on background a server",                               cli_parse_purge_server, NULL },
        {{},}
 }};