From: Remi Tricot-Le Breton Date: Wed, 18 Nov 2026 14:09:30 +0000 (+0100) Subject: BUG/MEDIUM: jwt: Use only JWT cert tree for jwt_verify X-Git-Url: https://git.kaiwu.me/web/%22data:,//%22%22?a=commitdiff_plain;h=607222c3569ba136d34ff44f3df2f67971db2f7e;p=haproxy.git BUG/MEDIUM: jwt: Use only JWT cert tree for jwt_verify The jwt_verify converter can either used a public key that was already stored in a dedicated tree built during init that did not require any lock, or use a certificate loaded in the ckch_store tree. The main issue in this second case is that we needed to take the ckch_lock during runtime but this spinlock is also taken during all CLI commands that modify the certificates (and can last a long time when a lot of chck instances are rebuilt for instance). We now store the public keys used by jwt_verify in the JWT certificate tree for all code paths instead of looking them up in the CKCH tree. We keep the tree entry in sync when certificates are loaded, replaced, removed, or when the "jwt" option is toggled through the CLI. In those cases, the lock on the JWT tree in only taken while the tree gets updated, during the ckch_store_replace call. A keyless entry is also created for certificates whose "jwt" option is disabled so that jwt_verify can still report JWT_VRFY_UNAVAIL_CERT (-7) instead of JWT_VRFY_UNKNOWN_CERT (-5). This patch relies on "MINOR: jwt: Add rwlock to jwt_cert_tree". It fixes the issue described in GitHub #3526 (and raised by some IA models as well). This patch can be backported up to 3.3. --- diff --git a/include/haproxy/jwt-t.h b/include/haproxy/jwt-t.h index 054d2df68..0ad750668 100644 --- a/include/haproxy/jwt-t.h +++ b/include/haproxy/jwt-t.h @@ -66,7 +66,7 @@ enum jwt_elt { }; struct jwt_cert_tree_entry { - EVP_PKEY *pubkey; + EVP_PKEY *pubkey; /* Public key used by jwt_verify */ struct ebmb_node node; char path[VAR_ARRAY]; }; diff --git a/include/haproxy/jwt.h b/include/haproxy/jwt.h index cf8410b94..32bf04eca 100644 --- a/include/haproxy/jwt.h +++ b/include/haproxy/jwt.h @@ -24,6 +24,7 @@ #include #include +#include #ifdef USE_OPENSSL enum jwt_alg jwt_parse_alg(const char *alg_str, unsigned int alg_len); @@ -33,6 +34,11 @@ int jwt_tree_load_cert(char *path, int pathlen, int tryload_cert, const char *fi enum jwt_vrfy_status jwt_verify(const struct buffer *token, const struct buffer *alg, const struct buffer *key, int is_x509); +int jwt_tree_create_entry_from_store(struct ckch_store *store, char **err); +int jwt_tree_delete_entry(char *path); +int jwt_tree_clear_entry(char *path); +int jwt_tree_get_pubkey(char *path, EVP_PKEY **pkey); + #endif /* USE_OPENSSL */ #endif /* _HAPROXY_JWT_H */ diff --git a/src/jwt.c b/src/jwt.c index dcece6f9f..a437a3202 100644 --- a/src/jwt.c +++ b/src/jwt.c @@ -139,6 +139,175 @@ int jwt_tokenize(const struct buffer *jwt, struct jwt_item *items, unsigned int return (ptr != jwt_end); } +/* + * Allocate a new entry for and insert it into the JWT certificate tree. + * is optional and may be NULL, which is used to mark a certificate as + * known to the JWT tree while making it unusable by the jwt_verify converter. + * The key is not stolen, a reference is taken on it, so that the caller keeps + * the ownership of its own reference. If an entry already exists for this path + * (for instance because it was kept around by jwt_tree_clear_entry() after a + * "del ssl jwt" command), its key is replaced in place instead of creating a + * duplicate entry, which also makes it possible to re-enable the "jwt" option + * on a certificate. + * Returns 0 on success, -1 on allocation failure (and fills if provided). + */ +static int jwt_tree_create_entry(char *path, int pathlen, EVP_PKEY *pubkey, char **err) +{ + struct jwt_cert_tree_entry *entry = NULL; + struct ebmb_node *eb = NULL; + + entry = calloc(1, sizeof(*entry) + pathlen + 1); + if (!entry) { + if (err) + memprintf(err, "%sunable to allocate memory (jwt_cert_tree_entry).\n", *err ? *err : ""); + return -1; + } + memcpy(entry->path, path, pathlen + 1); + + HA_RWLOCK_WRLOCK(JWT_LOCK, &jwt_tree_lock); + eb = ebst_insert(&jwt_cert_tree, &entry->node); + + if (eb != &entry->node) { + /* An entry already exists for this path. This happens when + * the "jwt" option was disabled then re-enabled on a + * certificate. Update the key of the existing entry in place. */ + ha_free(&entry); + entry = ebmb_entry(eb, struct jwt_cert_tree_entry, node); + } + + EVP_PKEY_free(entry->pubkey); + entry->pubkey = NULL; + if (pubkey) { + entry->pubkey = pubkey; + EVP_PKEY_up_ref(pubkey); + } + HA_RWLOCK_WRUNLOCK(JWT_LOCK, &jwt_tree_lock); + + return 0; +} + +/* + * Create (or update) the JWT tree entry matching the ckch_store . + * When the store's "jwt" option is enabled, the public key is extracted from + * the certificate stored in . An entry is created even when the option + * is disabled, but without any key: this way the certificate is known to the + * JWT tree and jwt_verify can return JWT_VRFY_UNAVAIL_CERT instead of + * JWT_VRFY_UNKNOWN_CERT. + * Returns 0 on success (or if nothing had to be done), non-zero on failure. + */ +int jwt_tree_create_entry_from_store(struct ckch_store *store, char **err) +{ + EVP_PKEY *pubkey = NULL; + int retval = 1; + + if (!store) + return 0; + + if (store->conf.jwt) { + /* X509_get_pubkey() returns an owned reference which is released + * below. Note that the entry creation takes its own reference on + * the key. */ + pubkey = X509_get_pubkey(store->data->cert); + } + + retval = jwt_tree_create_entry(store->path, strlen(store->path), pubkey, err); + EVP_PKEY_free(pubkey); + + return retval; +} + +/* + * Release the public key held by the JWT tree entry matching but keep + * the entry itself in the tree. This is used when the "jwt" option is disabled + * on a certificate ("del ssl jwt"): keeping an empty entry makes it possible + * for jwt_verify to return JWT_VRFY_UNAVAIL_CERT instead of + * JWT_VRFY_UNKNOWN_CERT. The key can later be restored by + * jwt_tree_create_entry(). + * Returns 0 (the entry may legitimately not exist). + */ +int jwt_tree_clear_entry(char *path) +{ + struct jwt_cert_tree_entry *entry = NULL; + struct ebmb_node *eb = NULL; + + HA_RWLOCK_WRLOCK(JWT_LOCK, &jwt_tree_lock); + eb = ebst_lookup(&jwt_cert_tree, path); + if (eb != NULL) { + entry = ebmb_entry(eb, struct jwt_cert_tree_entry, node); + EVP_PKEY_free(entry->pubkey); + entry->pubkey = NULL; + } + HA_RWLOCK_WRUNLOCK(JWT_LOCK, &jwt_tree_lock); + + return 0; +} + +/* Remove the JWT tree entry matching and release the reference it held + * on its public key. This must be used when the matching ckch_store is really + * being unloaded (certificate replaced or deleted) and not for temporary stores + * which are duplicates of an active one. + */ +int jwt_tree_delete_entry(char *path) +{ + struct jwt_cert_tree_entry *entry = NULL; + struct ebmb_node *eb = NULL; + + HA_RWLOCK_WRLOCK(JWT_LOCK, &jwt_tree_lock); + eb = ebst_lookup(&jwt_cert_tree, path); + if (eb != NULL) { + entry = ebmb_entry(eb, struct jwt_cert_tree_entry, node); + EVP_PKEY_free(entry->pubkey); + ebmb_delete(eb); + free(entry); + } + HA_RWLOCK_WRUNLOCK(JWT_LOCK, &jwt_tree_lock); + + return 0; +} + +/* + * Look for the JWT tree entry matching and return its public key. + * When is NULL, no key is returned and the function only checks that the + * public key is available for this entry (this is used to validate a + * certificate path at configuration parsing time). + * When a key is returned, a reference is taken on it and the caller is + * responsible for calling EVP_PKEY_free() on it. + * Returns 0 on success, JWT_VRFY_UNKNOWN_CERT if no entry exists for or + * JWT_VRFY_UNAVAIL_CERT if the entry exists but the public key is not + * available (e.g. "jwt" option disabled on the certificate). + */ +int jwt_tree_get_pubkey(char *path, EVP_PKEY **pkey) +{ + struct jwt_cert_tree_entry *entry = NULL; + struct ebmb_node *eb = NULL; + int retval = JWT_VRFY_UNKNOWN_CERT; + + HA_RWLOCK_RDLOCK(JWT_LOCK, &jwt_tree_lock); + eb = ebst_lookup(&jwt_cert_tree, path); + + if (!eb) + goto end; + + entry = ebmb_entry(eb, struct jwt_cert_tree_entry, node); + + if (!pkey) { + /* We don't want to extract the key, just to check whether it is + * available. */ + retval = entry->pubkey ? 0 : JWT_VRFY_UNAVAIL_CERT; + goto end; + } + + *pkey = entry->pubkey; + if (*pkey) + EVP_PKEY_up_ref(*pkey); + + retval = (*pkey ? 0 : JWT_VRFY_UNAVAIL_CERT); + +end: + HA_RWLOCK_RDUNLOCK(JWT_LOCK, &jwt_tree_lock); + return retval; +} + /* * Parse a public certificate and insert it into the jwt_cert_tree. * Returns 0 in case of success. @@ -146,15 +315,17 @@ int jwt_tokenize(const struct buffer *jwt, struct jwt_item *items, unsigned int int jwt_tree_load_cert(char *path, int pathlen, int tryload_cert, const char *file, int line, char **err) { int retval = -1; - struct jwt_cert_tree_entry *entry = NULL; EVP_PKEY *pubkey = NULL; BIO *bio = NULL; struct stat buf; struct ckch_store *store = NULL; + /* If an entry already exists for this path, the certificate was already + * declared (for instance in a crt-store or a crt-list). Respect its + * "jwt" option instead of enabling it implicitly: an entry without a + * public key makes jwt_verify return JWT_VRFY_UNAVAIL_CERT. */ HA_RWLOCK_RDLOCK(JWT_LOCK, &jwt_tree_lock); if (ebst_lookup(&jwt_cert_tree, path) != NULL) { - /* Entry already in the tree, nothing to do. */ HA_RWLOCK_RDUNLOCK(JWT_LOCK, &jwt_tree_lock); return 0; } @@ -175,27 +346,14 @@ int jwt_tree_load_cert(char *path, int pathlen, int tryload_cert, const char *fi * named crt-store). */ if (pubkey) { - entry = calloc(1, sizeof(*entry) + pathlen + 1); - if (!entry) { - memprintf(err, "%sunable to allocate memory (jwt_cert_tree_entry).\n", err && *err ? *err : ""); + retval = jwt_tree_create_entry(path, pathlen, pubkey, err); + if (retval < 0) goto end; - } - memcpy(entry->path, path, pathlen + 1); - - entry->pubkey = pubkey; - - HA_RWLOCK_WRLOCK(JWT_LOCK, &jwt_tree_lock); - if (ebst_insert(&jwt_cert_tree, &entry->node) != &entry->node) { - /* Another thread may have inserted the - * same entry in the meantime. It should - * not happen for now since this - * function is only called during conf - * parsing but better safe than sorry. */ - EVP_PKEY_free(pubkey); - pubkey = NULL; - ha_free(&entry); - } - HA_RWLOCK_WRUNLOCK(JWT_LOCK, &jwt_tree_lock); + + /* If entry creation went well, the pubkey + * reference counter will have been incremented + * already. */ + EVP_PKEY_free(pubkey); retval = 0; goto end; @@ -250,18 +408,11 @@ int jwt_tree_load_cert(char *path, int pathlen, int tryload_cert, const char *fi store->conf.jwt = 1; - retval = 0; + retval = jwt_tree_create_entry_from_store(store, err); HA_SPIN_UNLOCK(CKCH_LOCK, &ckch_lock); end: - if (retval) { - /* Some error happened during pubkey parsing, remove the already - * inserted node from the tree and free it. - */ - EVP_PKEY_free(pubkey); - free(entry); - } BIO_free(bio); return retval; @@ -389,7 +540,6 @@ jwt_jwsverify_rsa_ecdsa(const struct jwt_ctx *ctx, struct buffer *decoded_signat EVP_MD_CTX *evp_md_ctx; EVP_PKEY_CTX *pkey_ctx = NULL; enum jwt_vrfy_status retval = JWT_VRFY_KO; - struct jwt_cert_tree_entry *entry = NULL; int is_ecdsa = 0; int padding = RSA_PKCS1_PADDING; EVP_PKEY *pubkey = NULL; @@ -437,46 +587,10 @@ jwt_jwsverify_rsa_ecdsa(const struct jwt_ctx *ctx, struct buffer *decoded_signat if (!evp_md_ctx) return JWT_VRFY_OUT_OF_MEMORY; - if (ctx->is_x509) { - struct ckch_store *store = NULL; - if (!HA_SPIN_TRYLOCK(CKCH_LOCK, &ckch_lock)) { - - store = ckchs_lookup(ctx->key); - if (store) { - if (store->conf.jwt) { - /* Note: X509_get_pubkey() already returns an - * owned reference, don't take another one! - */ - pubkey = X509_get_pubkey(store->data->cert); - } else - retval = JWT_VRFY_UNAVAIL_CERT; - } - HA_SPIN_UNLOCK(CKCH_LOCK, &ckch_lock); - } - } else { - struct ebmb_node *eb = NULL; - /* Look for a public key in the JWT tree. We can keep a read - * lock because the tree itself is not changed and the only - * operation on the entry is EVP_PKEY_up_ref called on the pkey - * field which is atomic. */ - HA_RWLOCK_RDLOCK(JWT_LOCK, &jwt_tree_lock); - eb = ebst_lookup(&jwt_cert_tree, ctx->key); - - if (eb) { - entry = ebmb_entry(eb, struct jwt_cert_tree_entry, node); - - pubkey = entry->pubkey; - if (pubkey) - EVP_PKEY_up_ref(pubkey); - } - HA_RWLOCK_RDUNLOCK(JWT_LOCK, &jwt_tree_lock); - } - - if (!pubkey) { - if (!retval) - retval = JWT_VRFY_UNKNOWN_CERT; + /* Look for a public key in the JWT tree. */ + retval = jwt_tree_get_pubkey(ctx->key, &pubkey); + if (retval) goto end; - } /* * ECXXX signatures are a direct concatenation of the (R, S) pair and @@ -625,6 +739,7 @@ error: } + static void jwt_deinit(void) { struct ebmb_node *node = NULL; diff --git a/src/ssl_ckch.c b/src/ssl_ckch.c index fb1b70668..bf73c6263 100644 --- a/src/ssl_ckch.c +++ b/src/ssl_ckch.c @@ -42,6 +42,7 @@ #include #include #include +#include /* Uncommitted CKCH transaction */ @@ -1013,6 +1014,14 @@ void ckch_store_free(struct ckch_store *store) list_for_each_entry_safe(inst, inst_s, &store->ckch_inst, by_ckchs) { ckch_inst_free(inst); } + + /* If the store was actually inserted in the tree we need to remove the + * corresponding JWT tree entry as well. Temporary stores (duplicated for + * a transaction, for instance) share their path with the active store + * and must not remove its JWT entry. */ + if (store->node.node.leaf_p) + jwt_tree_delete_entry(store->path); + ebmb_delete(&store->node); ssl_sock_free_cert_key_and_chain_contents(store->data); @@ -1178,8 +1187,15 @@ struct ckch_store *ckch_store_new_load_files_path(char *path, char **err) ckchs->conf.used = CKCH_CONF_SET_EMPTY; - /* insert into the ckchs tree */ memcpy(ckchs->path, path, strlen(path) + 1); + + /* Insert pubkey into JWT-specific tree */ + if (jwt_tree_create_entry_from_store(ckchs, err)) { + memprintf(err,"%scould not create jwt tree entry for '%s'.", err && *err ? *err : "", path); + goto end; + } + + /* insert into the ckchs tree */ ebst_insert(&ckchs_tree, &ckchs->node); return ckchs; @@ -1224,8 +1240,15 @@ struct ckch_store *ckch_store_new_load_files_conf(char *name, struct ckch_conf * conf->crt = tmpcrt; - /* insert into the ckchs tree */ memcpy(ckchs->path, name, strlen(name) + 1); + + /* Insert pubkey into JWT-specific tree */ + if (jwt_tree_create_entry_from_store(ckchs, err)) { + memprintf(err,"%scould not create jwt tree entry for '%s'.", err && *err ? *err : "", ckchs->path); + goto end; + } + + /* insert into the ckchs tree */ ebst_insert(&ckchs_tree, &ckchs->node); return ckchs; @@ -2533,6 +2556,7 @@ static inline int cli_set_ssl_jwt(char **args, struct appctx *appctx, int value) { struct ckch_store *ckchs; char *err = NULL; + int oldvalue; if (!cli_has_level(appctx, ACCESS_LVL_ADMIN)) return cli_err(appctx, "Can't allocate memory!\n"); @@ -2549,8 +2573,19 @@ static inline int cli_set_ssl_jwt(char **args, struct appctx *appctx, int value) if ((ckchs = ckchs_lookup(args[3])) == NULL) goto error; + oldvalue = ckchs->conf.jwt; ckchs->conf.jwt = value; + if (value) { + if (jwt_tree_create_entry_from_store(ckchs, &err)) { + ckchs->conf.jwt = oldvalue; + HA_SPIN_UNLOCK(CKCH_LOCK, &ckch_lock); + return cli_dynerr(appctx, memprintf(&err, "%sAdding certificate '%s' to JWT tree failed", err ? err : "\n", args[3])); + } + } else { + jwt_tree_clear_entry(args[3]); + } + HA_SPIN_UNLOCK(CKCH_LOCK, &ckch_lock); return 0; @@ -3068,6 +3103,11 @@ void ckch_store_replace(struct ckch_store *old_ckchs, struct ckch_store *new_ckc } ckch_store_free(old_ckchs); + + /* The old entry should have been removed in chck_store_free, create a + * new entry altogether. */ + jwt_tree_create_entry_from_store(new_ckchs, NULL); + ebst_insert(&ckchs_tree, &new_ckchs->node); } @@ -3522,6 +3562,10 @@ static int cli_parse_del_cert(char **args, char *payload, struct appctx *appctx, } ebmb_delete(&store->node); + /* The store is removed from the ckch tree, remove its JWT entry too. + * Note that a store in use for JWT validation cannot be deleted, but an + * empty entry may remain after a "del ssl jwt" command. */ + jwt_tree_delete_entry(store->path); ckch_store_free(store); memprintf(&err, "Certificate '%s' deleted!\n", filename); @@ -5577,6 +5621,17 @@ static int crtstore_parse_load(char **args, int section_type, struct proxy *curp goto out; } + /* Create the JWT tree entry. It is created even when the "jwt" option is + * not enabled, so that the certificate is known to the JWT tree and + * JWT_VRFY_UNAVAIL_CERT can be returned instead of + * JWT_VRFY_UNKNOWN_CERT. */ + if (jwt_tree_create_entry_from_store(c, err)) { + memprintf(err,"parsing [%s:%d] : '%s' in section 'crt-store': could not create jwt tree entry for '%s'.", + file, linenum, args[0], c->path); + err_code |= ERR_ALERT | ERR_FATAL; + goto out; + } + out: /* free ckch_conf content */ if (err_code & ERR_FATAL)