(or compiled without the threads support), these callbacks will be silently
ignored (in this case, global.nbthread will be always equal to one).
+For a proxy dynamically created at runtime (with the "add backend" CLI
+command), the lifecycle follows the same stages: the check and init callbacks
+run on the thread performing the creation, the per-thread callbacks are
+deferred to every thread, and the deinit callbacks run on the thread
+performing the deletion, once every thread ran its per-thread callbacks.
+
3.4. HANDLING THE STREAMS ACTIVITY
-----------------------------------
int flt_has_explicit_config(const struct proxy *px);
int flt_copy_instances(struct proxy *px, const struct proxy *defpx);
void flt_free_instances(struct proxy *px);
+
+int flt_start_new_proxy(struct proxy *px);
+int flt_init_new_proxy(struct proxy *px);
+void flt_deinit_new_proxy(struct proxy *px);
+
int flt_init_class_refs(struct proxy *px);
int flt_add_implicit_instance(struct proxy *px, struct filter_class *cls, const char *id,
char **args, const char *file, int line);
--- /dev/null
+varnishtest "Dynamic backends inherit and activate their filter instances"
+
+feature cmd "$HAPROXY_PROGRAM -cc 'feature(LUA) && feature(THREAD)'"
+feature ignore_unknown_macro
+
+# The instances of the defaults section are inherited by the dynamic
+# backends, finalized at creation time and initialized on every thread: the
+# per-thread Lua states of the D-scoped instances are ready when the streams
+# are attached on both threads.
+server s1 {
+ # Before the dynamic backend exists, the request falls through the
+ # use_backend rule to the default backend, which has no filter.
+ rxreq
+ expect req.http.x-probe-order == <undef>
+ txresp -body "no filter yet"
+ close
+ accept
+
+ rxreq
+ expect req.http.x-probe-order == "D.a;D.b;"
+ expect req.http.x-probe-threads == "D.a:1;D.b:1;"
+ expect req.http.x-probe-instances == "D.a:2;D.b:1;"
+ expect req.http.x-probe-errors == <undef>
+ txresp -body "dyn thread one"
+ close
+ accept
+
+ rxreq
+ expect req.http.x-probe-order == "D.a;D.b;"
+ expect req.http.x-probe-threads == "D.a:2;D.b:2;"
+ expect req.http.x-probe-instances == "D.a:2;D.b:1;"
+ expect req.http.x-probe-errors == <undef>
+ txresp -body "dyn thread two"
+ close
+ accept
+
+ rxreq
+ expect req.http.x-probe-order == "D.a;D.b;"
+ expect req.http.x-probe-threads == "D.a:1;D.b:1;"
+ expect req.http.x-probe-instances == "D.a:4;D.b:3;"
+ expect req.http.x-probe-errors == <undef>
+ txresp -body "dyn thread one again"
+} -start
+
+haproxy h1 -conf {
+ global
+ chroot /
+ nbthread 2
+ thread-groups 1
+ tune.lua.bool-sample-conversion normal
+ lua-load-per-thread "${testdir}/lua-probe.lua"
+
+ defaults plain
+ mode http
+ timeout connect "${HAPROXY_TEST_TIMEOUT-5s}"
+ timeout client "${HAPROXY_TEST_TIMEOUT-5s}"
+ timeout server "${HAPROXY_TEST_TIMEOUT-5s}"
+
+ # The instances of this defaults section are inherited and activated by
+ # the dynamic backends created from it. probe-c is explicitly disabled.
+ # The global sequence changes the attachment order (b then a), the
+ # response sequence the response evaluation order, the request order
+ # keeps the declaration order.
+ defaults dynbe from plain
+ filter-config lua id probe-a enabled D a
+ filter-config lua id probe-b enabled D b
+ filter-config lua id probe-c enabled D c
+ filter-sequence global lua/probe-b:before(lua/probe-a)
+ filter-sequence response lua/probe-b:before(lua/probe-a)
+ filter-disable lua/probe-c
+
+ frontend one from plain
+ bind "fd@${one}" thread 1
+ use_backend %[req.hdr(x-be)]
+ default_backend be
+
+ frontend two from plain
+ bind "fd@${two}" thread 2
+ use_backend %[req.hdr(x-be)]
+ default_backend be
+
+ backend be from plain
+ server s1 ${s1_addr}:${s1_port}
+} -start
+
+# The dynamic backend does not exist yet, the request is served by the
+# default backend.
+client c1 -connect ${h1_one_sock} {
+ txreq -hdr "x-be: dyn"
+ rxresp
+ expect resp.status == 200
+ expect resp.body == "no filter yet"
+} -run
+
+haproxy h1 -cli {
+ send "add backend dyn from dynbe"
+ expect ~ "New backend registered."
+
+ send "add server dyn/s1 ${s1_addr}:${s1_port}"
+ expect ~ "New server registered."
+
+ send "enable server dyn/s1"
+ expect ~ ".*"
+
+ send "publish backend dyn"
+ expect ~ "Backend published."
+
+ # A second dynamic backend, kept unpublished and empty so it can be
+ # deleted: its per-thread filter contexts are deinitialized on every
+ # thread during the deletion.
+ send "add backend dyn2 from dynbe"
+ expect ~ "New backend registered."
+}
+
+# The inherited instances are active on the first thread, with the response
+# evaluation order from the inherited sequence.
+client c2 -connect ${h1_one_sock} {
+ txreq -hdr "x-be: dyn"
+ rxresp
+ expect resp.status == 200
+ expect resp.http.x-probe-order == "D.b;D.a;"
+ expect resp.body == "dyn thread one"
+} -run
+
+# ... and on the second thread.
+client c3 -connect ${h1_two_sock} {
+ txreq -hdr "x-be: dyn"
+ rxresp
+ expect resp.status == 200
+ expect resp.http.x-probe-order == "D.b;D.a;"
+ expect resp.body == "dyn thread two"
+} -run
+
+haproxy h1 -cli {
+ send "del backend dyn2"
+ expect ~ "Backend deleted."
+}
+
+# The deletion of dyn2 released the per-thread contexts of every thread
+# without altering the ones of dyn.
+client c4 -connect ${h1_one_sock} {
+ txreq -hdr "x-be: dyn"
+ rxresp
+ expect resp.status == 200
+ expect resp.body == "dyn thread one again"
+} -run
varnishtest "Enforce the legacy filter mode on the proxies without declaration"
-# Regression test for the "filter-api downgrade-to-legacy" global option:
-# the proxies without explicit filter declaration must use the legacy filter
-# mode, the ones with an explicit "filter-*" declaration must keep the new mode.
+# Regression test for the "filter-api downgrade-to-legacy" global option: the proxies
+# without explicit filter declaration must use the legacy filter mode, the
+# ones with an explicit "filter-*" declaration must keep the new mode. The
+# backends dynamically created with the "add backend" CLI command follow
+# the same rule.
feature ignore_unknown_macro
server s1 {
rxreq
txresp -hdr "Content-Type: text/plain" -bodylen 500
-} -repeat 2 -start
+} -repeat 3 -start
haproxy h1 -conf {
global
filter-api downgrade-to-legacy
+ chroot /
- defaults
+ defaults plain
mode http
timeout connect "${HAPROXY_TEST_TIMEOUT-5s}"
timeout client "${HAPROXY_TEST_TIMEOUT-5s}"
timeout server "${HAPROXY_TEST_TIMEOUT-5s}"
+ # The instances of this defaults section, and the compression options,
+ # are inherited by the backends created at runtime with "add backend".
+ # Without explicit filter declaration, they work in legacy mode.
+ defaults dynbe from plain
+ compression algo gzip
+ compression type text/plain
+
# No filter declaration: with the legacy filter mode enforced globally,
# the implicit declaration of the compression filter is handled in the
# legacy mode, it must be operational.
- listen li-implicit
+ listen li-implicit from plain
bind "fd@${li1}"
compression algo gzip
compression type text/plain
# declaration always wins over the mode enforced globally, and the
# explicit instance replaces the implicit one. It is not enabled, so
# the compression is not performed.
- listen li-explicit
+ listen li-explicit from plain
bind "fd@${li2}"
filter-config http-comp-res
compression algo gzip
compression type text/plain
server s1 ${s1_addr}:${s1_port}
+
+ frontend fe from plain
+ bind "fd@${fe}"
+ use_backend %[req.hdr(x-be)]
+ default_backend be
+
+ backend be from plain
+ server s1 ${s1_addr}:${s1_port}
} -start
client c-implicit -connect ${h1_li1_sock} {
rxresp
expect resp.http.content-encoding == <undef>
} -run
+
+# The dynamic backend inherits the compression options of the dynbe
+# defaults section. Without explicit filter declaration, the legacy filter
+# mode is enforced, the implicit compression is handled in the legacy
+# mode.
+haproxy h1 -cli {
+ send "add backend dyn from dynbe"
+ expect ~ "New backend registered."
+
+ send "add server dyn/s1 ${s1_addr}:${s1_port}"
+ expect ~ "New server registered."
+
+ send "enable server dyn/s1"
+ expect ~ ".*"
+
+ send "publish backend dyn"
+ expect ~ "Backend published."
+}
+
+client c-dyn -connect ${h1_fe_sock} {
+ txreq -url /dyn -hdr "x-be: dyn" -hdr "Accept-Encoding: gzip"
+ rxresp
+ expect resp.http.content-encoding == gzip
+} -run
#include <haproxy/filters.h>
#include <haproxy/flt_decomp.h>
#include <haproxy/flt_http_comp.h>
+#include <haproxy/global.h>
#include <haproxy/http_ana.h>
#include <haproxy/http_htx.h>
#include <haproxy/htx.h>
#include <haproxy/namespace.h>
#include <haproxy/proxy.h>
#include <haproxy/stream.h>
+#include <haproxy/thread.h>
#include <haproxy/tools.h>
+#include <haproxy/twork.h>
#include <haproxy/trace.h>
return 0;
}
+/**************************************************************************
+ * Filter instances of proxies created at runtime
+ *************************************************************************/
+
+/* Deferred per-thread work item for a proxy created at runtime: runs
+ * flt_init_per_thread() for the proxy <arg.ptr>, in the context of the
+ * thread the item was posted to, as the filters may initialize thread-local
+ * variables. The ERR_* flags are returned to the completion callback, but
+ * only for reporting: the creation of the proxy cannot be aborted anymore.
+ */
+static union twork_arg flt_twork_init_per_thread(union twork_arg arg)
+{
+ struct proxy *px = arg.ptr;
+ int err_code;
+
+ err_code = flt_init_per_thread(px);
+ if (err_code & (ERR_ABORT|ERR_FATAL))
+ ha_alert("Failed to initialize the per-thread filters of proxy '%s' for thread %u.\n",
+ px->id, tid);
+ return twk_u32(err_code);
+}
+
+/* Deferred per-thread work item for a proxy created at runtime: runs
+ * flt_deinit_per_thread() for the proxy <arg.ptr>, in the context of the
+ * thread the item was posted to, as the filters may release thread-local
+ * variables.
+ */
+static union twork_arg flt_twork_deinit_per_thread(union twork_arg arg)
+{
+ flt_deinit_per_thread(arg.ptr);
+ return twk_u32(ERR_NONE);
+}
+
+/* Completion callback of the deferred per-thread work items of a proxy
+ * created at runtime, running on the thread that posted them: releases the
+ * reference the item held on the proxy, so the completion of the last item
+ * marks the completion of the per-thread callbacks of every thread.
+ */
+static void flt_twork_done(union twork_arg arg, union twork_arg ret)
+{
+ proxy_drop(arg.ptr);
+}
+
+/* Finalizes the filter instances of the proxy <px> created at runtime (for
+ * now only with the "add backend" CLI command, after the instances inherited
+ * from its defaults section were copied): the parse callback of every
+ * instance is called to produce its filter configuration, the
+ * filter-enable/filter-disable and filter-sequence directives are applied,
+ * and the three evaluation orders are flattened. This mirrors the pre-check
+ * stage of the startup, so flt_start_new_proxy() must be called before
+ * proxy_finalize(), while the threads are isolated, so that the filter
+ * analyzers are enabled if the proxy has filters.
+ * Returns a combination of ERR_* flags, ERR_NONE on success.
+ */
+int flt_start_new_proxy(struct proxy *px)
+{
+ int err_code = ERR_NONE;
+
+ /* The filter mode of a new proxy may depend on the globally requested
+ * one, apply it before finalizing the filter instances.
+ */
+ flt_apply_default_mode(px);
+
+ err_code |= flt_precheck_instances(px);
+ if (!(err_code & (ERR_ABORT|ERR_FATAL)))
+ err_code |= flt_enable_filters(px);
+ if (!(err_code & (ERR_ABORT|ERR_FATAL)))
+ err_code |= flt_apply_sequences(px);
+
+ if (err_code & (ERR_ABORT|ERR_FATAL)) {
+ ha_alert("Failed to finalize the filter instances of proxy '%s'.\n",
+ px->id);
+ return err_code;
+ }
+
+ /* Produce the flat evaluation orders and release the configuration
+ * structures
+ */
+ flt_flatten_instances(px);
+ return ERR_NONE;
+}
+
+/* Initializes the filters of the proxy <px> created at runtime, mirroring the
+ * post-check and per-thread initialization stages of the startup:
+ *
+ * - the 'init' callbacks run on the main thread during the startup, before
+ * the threads are started, so they run here on the thread performing the
+ * creation, with all the other threads isolated;
+ *
+ * - the 'init_per_thread' callbacks run on every thread during the startup,
+ * as the filters may initialize thread-local variables. The other threads
+ * being parked by the thread isolation, each callback is deferred to its
+ * thread with a twork item: the items are queued while the threads are
+ * still parked and run as their first pending work when they resume. The
+ * new proxy is not reachable by the switching rules until it is published
+ * with "publish backend", which happens after the items completed; only
+ * "force-be-switch" could select it in-between.
+ *
+ * Each item holds a reference on the proxy, so the proxy cannot be destroyed
+ * before its per-thread callbacks ran (e.g. by a concurrent "del backend").
+ * flt_init_new_proxy() must be called after flt_start_new_proxy() and the
+ * post-proxy-check callbacks, while the threads are isolated.
+ * Returns a combination of ERR_* flags, ERR_NONE on success.
+ */
+int flt_init_new_proxy(struct proxy *px)
+{
+ int err_code = ERR_NONE;
+ int t;
+
+ BUG_ON(!thread_isolated());
+
+ err_code |= flt_init(px);
+ if (err_code & (ERR_ABORT|ERR_FATAL)) {
+ ha_alert("Failed to initialize the filters of proxy '%s'.\n", px->id);
+ return err_code;
+ }
+
+ /* nothing to defer without filter */
+ if (LIST_ISEMPTY(&px->filter_instances) && LIST_ISEMPTY(&px->filter_configs))
+ return ERR_NONE;
+
+ for (t = 0; t < global.nbthread; t++) {
+ proxy_take(px);
+ if (twork_call_on(NULL, t, flt_twork_init_per_thread, twk_ptr(px),
+ flt_twork_done))
+ continue;
+
+ /* the item could not be queued: release the reference taken
+ * for it and report the failure. The creation will be aborted,
+ * so also defer the deinitialization of the threads already
+ * notified, to balance their per-thread initialization.
+ */
+ proxy_drop(px);
+ err_code |= ERR_ALERT|ERR_FATAL;
+ ha_alert("Failed to defer the per-thread filter initialization of proxy '%s' for thread %u.\n",
+ px->id, t);
+ while (t-- > 0) {
+ proxy_take(px);
+ if (!twork_call_on(NULL, t, flt_twork_deinit_per_thread,
+ twk_ptr(px), flt_twork_done))
+ proxy_drop(px);
+ }
+ return err_code;
+ }
+ return ERR_NONE;
+}
+
+/* Deinitializes the per-thread filters of the proxy <px> created at runtime
+ * (for now only with the "del backend" CLI command), mirroring the
+ * per-thread deinitialization stage of the shutdown: the
+ * 'deinit_per_thread' callbacks are deferred to each thread with twork
+ * items, queued while the threads are still parked by the thread isolation
+ * of the deletion. The callbacks may release thread-local variables or
+ * per-thread resources (e.g. references in the per-thread Lua states), so
+ * they must complete before the proxy is destroyed: each item holds a
+ * reference on the proxy, and the caller only releases the creation
+ * reference, so the proxy memory is freed by the completion of the last
+ * item, once every thread ran its callbacks.
+ */
+void flt_deinit_new_proxy(struct proxy *px)
+{
+ int t;
+
+ BUG_ON(!thread_isolated());
+
+ /* nothing to defer without filter */
+ if (LIST_ISEMPTY(&px->filter_instances) && LIST_ISEMPTY(&px->filter_configs))
+ return;
+
+ for (t = 0; t < global.nbthread; t++) {
+ proxy_take(px);
+ if (twork_call_on(NULL, t, flt_twork_deinit_per_thread, twk_ptr(px),
+ flt_twork_done))
+ continue;
+
+ /* the item could not be queued: release the reference taken
+ * for it. The per-thread resources of this thread, if any,
+ * leak with the proxy, which is about to be destroyed.
+ */
+ proxy_drop(px);
+ ha_alert("Failed to defer the per-thread filter deinitialization of proxy '%s' for thread %u.\n",
+ px->id, t);
+ }
+}
+
/* Dump classes in global registration order or in a channel's placement tree. */
static void cli_dump_flt_classes(struct buffer *out, struct list *classes, unsigned int side, int *first)
{
if (proxy_ref_defaults(px, defpx, &msg))
goto err;
+ /* Finalize the filter instances inherited from <defpx>, before the
+ * proxy finalization so the filter analyzers are enabled if needed.
+ */
+ err_code |= flt_start_new_proxy(px);
+ if (err_code & (ERR_ABORT|ERR_FATAL)) {
+ memprintf(&msg, "failed to finalize the filter instances");
+ goto err;
+ }
+
proxy_init_per_thr(px);
if (proxy_finalize(px, &err_code))
goto err;
}
+ /* Initialize the filters on every thread, after the post-proxy-check
+ * callbacks which validate the instances, while the threads are still
+ * isolated.
+ */
+ err_code |= flt_init_new_proxy(px);
+ if (err_code & (ERR_ABORT|ERR_FATAL)) {
+ memprintf(&msg, "failed to initialize the filters");
+ goto err;
+ }
+
px->flags |= PR_FL_BE_UNPUBLISHED;
if (!stats_allocate_proxy_counters_internal(&px->extra_counters_be,
px->flags |= PR_FL_DELETED;
+ /* Deinitialize the per-thread filters in the context of every thread,
+ * before the proxy is destroyed, while the threads are still isolated.
+ */
+ flt_deinit_new_proxy(px);
+
thread_release();
ha_notice("Backend deleted.\n");
projects / haproxy.git / commitdiff