From: Olivier Houchard Date: Thu, 30 Jul 2026 17:42:17 +0000 (+0200) Subject: OPTIM: fd: Make each thread group's FD table NUMA-local X-Git-Tag: v3.5-dev10~93 X-Git-Url: https://git.kaiwu.me/$%7BGITURL%7D$%7Bcid%7D?a=commitdiff_plain;h=4312f13a2;p=haproxy.git OPTIM: fd: Make each thread group's FD table NUMA-local With per-thread-group FD tables, the tables are allocated and zeroed, hence faulted in, by the main thread, and the kernel's first-touch placement then places every group's table on the main thread's NUMA node, making the other groups' fdtab accesses, including the atomics done for every FD event, run on remote memory. Allocate the tables with mmap() instead, so that their pages are only faulted in when they are first written, and make these first writes come from the right CPUs: the boot thread is bound to group 0's CPUs before any FD is inserted, and to each other group's CPUs around that group's copy of the boot table, so that every table stays local to its own node. Each group's slice is padded to the kernel's page size so that no page is shared between two groups. Only matters with per-group FD tables. --- diff --git a/src/fd.c b/src/fd.c index 60dbb1a85..5544d2105 100644 --- a/src/fd.c +++ b/src/fd.c @@ -78,6 +78,7 @@ #include #include #include +#include #include #include #include @@ -94,12 +95,14 @@ #include #include #include +#include #include #include THREAD_LOCAL struct fdtab *fdtab = NULL; /* array of all the file descriptors */ static struct fdtab *_fdtab; +static size_t fdtab_alloc_size; /* size of the mmap() area backing _fdtab, for munmap() */ THREAD_LOCAL struct polled_mask *polled_mask = NULL; /* Array for the polled_mask of each fd */ static struct polled_mask *_polled_mask; int totalconn; /* total # of terminated sessions */ @@ -1215,24 +1218,39 @@ static void free_pollers_per_thread() */ int init_pollers() { + size_t fdtab_stride; int p; struct poller *bp; int nbfdtab; + ulong page_size; if (global.tune.options & GTUNE_NO_TG_FD_SHARING) { nbfdtab = global.nbtgroups; } else nbfdtab = 1; - /* always provide an aligned fdtab */ - if ((_fdtab = ha_aligned_zalloc(64, array_size_or_fail(global.maxsock, nbfdtab * sizeof(*fdtab)))) == NULL) { + page_size = sysconf(_SC_PAGESIZE); + /* Always provide an aligned fdtab, allocated with mmap() and not + * with a zeroing allocator on purpose: the pages must only be + * faulted in when they are first written, from each group's CPUs, + * so that the kernel's first-touch placement keeps every group's + * table local to its own NUMA node (see below and + * setup_extra_threads()). Each thread group's table starts on a + * page boundary, so that no page is shared between two groups. + */ + fdtab_stride = (array_size_or_fail(global.maxsock, sizeof(*fdtab)) + page_size - 1) & ~(page_size - 1); + fdtab_alloc_size = array_size_or_fail(nbfdtab, fdtab_stride); + _fdtab = mmap(NULL, fdtab_alloc_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (_fdtab == MAP_FAILED) { + _fdtab = NULL; ha_alert("Not enough memory to allocate %d entries for fdtab!\n", global.maxsock); goto fail_tab; } for (p = 0; p < global.nbtgroups; p++) { if (global.tune.options & GTUNE_NO_TG_FD_SHARING) { - ha_tgroup_ctx[p].fdtab = (void *)((char *)(void *)_fdtab + p * global.maxsock * sizeof(*fdtab)); + ha_tgroup_ctx[p].fdtab = (void *)((char *)(void *)_fdtab + p * fdtab_stride); } else ha_tgroup_ctx[p].fdtab = _fdtab; } @@ -1240,7 +1258,14 @@ int init_pollers() * Immediately set the fdtab for our thread, as we'll need it soon */ fdtab = ha_tgroup_ctx[0].fdtab; - vma_set_name(_fdtab, global.maxsock * sizeof(*fdtab) * nbfdtab, "fd", "fdtab"); + + /* The boot thread is thread 1 of group 0: bind it to group 0's CPUs + * before any FD is inserted, so that its writes are what place group + * 0's table on that group's NUMA node. The other groups' tables are + * placed when they are copied, see setup_extra_threads(). + */ + thread_bind_to_cpu_map(0, 0); + vma_set_name(_fdtab, fdtab_alloc_size, "fd", "fdtab"); if ((_polled_mask = calloc(global.maxsock, nbfdtab * sizeof(*polled_mask))) == NULL) { ha_alert("Not enough memory to allocate %d entries for polled_mask!\n", global.maxsock); @@ -1295,7 +1320,7 @@ redo: free(_polled_mask); _polled_mask = polled_mask = NULL; fail_polledmask: - ha_aligned_free(_fdtab); + munmap(_fdtab, fdtab_alloc_size); _fdtab = fdtab = NULL; fail_tab: return 0; @@ -1319,7 +1344,7 @@ void deinit_pollers() { /* free the backing areas, not the (possibly per-group) thread-local * pointers which may not point to the start of the allocations. */ - ha_aligned_free(_fdtab); + munmap(_fdtab, fdtab_alloc_size); _fdtab = fdtab = NULL; ha_free(&_polled_mask); polled_mask = NULL; diff --git a/src/thread.c b/src/thread.c index 431c0eb5d..aacae3c26 100644 --- a/src/thread.c +++ b/src/thread.c @@ -281,19 +281,28 @@ void setup_extra_threads(void *(*handler)(void *)) /* the startup thread will be thread 1 */ ha_pthread[0] = pthread_self(); + /* The FD tables were allocated with mmap() and are still unfaulted + * (see init_pollers()): copy each extra group's table now, when it + * holds the FDs common to all groups, with the boot thread bound to + * that group's CPUs so that the kernel's first-touch placement keeps + * it local to the group's NUMA node. The boot thread is finally + * bound back to its own group's CPUs, as init_pollers() had set. + */ + if (global.tune.options & GTUNE_NO_TG_FD_SHARING) { + for (i = 1; i < global.nbtgroups; i++) { + thread_bind_to_cpu_map(i, 0); + memcpy(ha_tgroup_ctx[i].fdtab, ha_tgroup_ctx[0].fdtab, + global.maxsock * sizeof(*fdtab)); + } + } + thread_bind_to_cpu_map(0, 0); + /* Create one initial thread for each extra thread group. These will * each be responsible for creating their own extra threads. The first * group's initial thread is the current thread. */ for (i = 1; i < global.nbtgroups; i++) { ha_tgroup_info[i].start = handler; - /* - * Copy the fdtab to the new thread group, it should contain - * file descriptors that are common to all thread groups - */ - if (global.tune.options & GTUNE_NO_TG_FD_SHARING) - memcpy(ha_tgroup_ctx[i].fdtab, ha_tgroup_ctx[0].fdtab, - global.maxsock * sizeof(*fdtab)); pthread_create(&ha_pthread[ha_tgroup_info[i].base], NULL, &start_extra_tgroup_threads, &ha_tgroup_info[i]); }