[PATCH 2 of 6] SSL: object caching
Aleksei Bavshin
a.bavshin at nginx.com
Mon Aug 12 20:52:31 UTC 2024
On 7/23/2024 12:30 PM, Mini Hawthorne wrote:
> # HG changeset patch
> # User Mini Hawthorne <mini at f5.com>
> # Date 1721762842 0
> # Tue Jul 23 19:27:22 2024 +0000
> # Node ID 8eee61e223bb7cb7475e50b866fd6b9a83fa5fa0
> # Parent 59ac183dfee8e9641563e043eb19480d91dd7cc0
> SSL: object caching.
>
> Added ngx_openssl_cache_module, which indexes a type-aware object cache.
> It maps an id to a unique instance, and provides references to it, which
> are dropped when the cycle's pool is destroyed. Also, for those objects
> that can be cached, valid references may be pulled from cycle->old_cycle.
>
> The cache will be used in subsequent patches.
>
> diff --git a/auto/modules b/auto/modules
> --- a/auto/modules
> +++ b/auto/modules
> @@ -1307,10 +1307,11 @@ fi
>
> if [ $USE_OPENSSL = YES ]; then
> ngx_module_type=CORE
> - ngx_module_name=ngx_openssl_module
> + ngx_module_name="ngx_openssl_module ngx_openssl_cache_module"
> ngx_module_incs=
> ngx_module_deps=src/event/ngx_event_openssl.h
> ngx_module_srcs="src/event/ngx_event_openssl.c
> + src/event/ngx_event_openssl_cache.c
> src/event/ngx_event_openssl_stapling.c"
> ngx_module_libs=
> ngx_module_link=YES
> diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h
> --- a/src/event/ngx_event_openssl.h
> +++ b/src/event/ngx_event_openssl.h
> @@ -83,7 +83,8 @@
> #endif
>
>
> -typedef struct ngx_ssl_ocsp_s ngx_ssl_ocsp_t;
> +typedef struct ngx_ssl_ocsp_s ngx_ssl_ocsp_t;
> +typedef struct ngx_ssl_cache_type_s ngx_ssl_cache_type_t;
>
>
> typedef struct {
> @@ -233,6 +234,9 @@ ngx_int_t ngx_ssl_ocsp_get_status(ngx_co
> void ngx_ssl_ocsp_cleanup(ngx_connection_t *c);
> ngx_int_t ngx_ssl_ocsp_cache_init(ngx_shm_zone_t *shm_zone, void *data);
>
> +void *ngx_ssl_cache_fetch(ngx_cycle_t *cycle, ngx_pool_t *pool,
> + ngx_ssl_cache_type_t *type, char **err, ngx_str_t *id, void *data);
> +
> ngx_array_t *ngx_ssl_read_password_file(ngx_conf_t *cf, ngx_str_t *file);
> ngx_array_t *ngx_ssl_preserve_passwords(ngx_conf_t *cf,
> ngx_array_t *passwords);
> diff --git a/src/event/ngx_event_openssl_cache.c b/src/event/ngx_event_openssl_cache.c
> new file mode 100644
> --- /dev/null
> +++ b/src/event/ngx_event_openssl_cache.c
> @@ -0,0 +1,277 @@
> +
> +/*
> + * Copyright (C) Nginx, Inc.
> + */
> +
> +
> +#include <ngx_config.h>
> +#include <ngx_core.h>
> +#include <ngx_event.h>
> +
> +
> +typedef struct {
> + ngx_rbtree_node_t node;
> + ngx_str_t id;
> +
> + ngx_ssl_cache_type_t *type;
> + void *value;
> +} ngx_ssl_cache_node_t;
> +
> +
> +typedef void *(*ngx_ssl_cache_create_pt)(ngx_str_t *id, char **err, void *data);
> +typedef void (*ngx_ssl_cache_free_pt)(void *data);
> +typedef void *(*ngx_ssl_cache_ref_pt)(char **err, void *data);
> +
> +
> +struct ngx_ssl_cache_type_s {
> + const char *name;
> +
> + ngx_ssl_cache_create_pt create;
> + ngx_ssl_cache_free_pt free;
> + ngx_ssl_cache_ref_pt ref;
> +};
> +
> +
> +typedef struct {
> + ngx_rbtree_t rbtree;
> + ngx_rbtree_node_t sentinel;
> +} ngx_ssl_cache_t;
> +
> +
> +static void *ngx_openssl_cache_create_conf(ngx_cycle_t *cycle);
> +static void ngx_ssl_cache_cleanup(void *data);
> +static void ngx_ssl_cache_node_insert(ngx_rbtree_node_t *temp,
> + ngx_rbtree_node_t *node, ngx_rbtree_node_t *sentinel);
> +
> +static ngx_ssl_cache_node_t *ngx_ssl_cache_lookup(ngx_ssl_cache_t *cache,
> + ngx_ssl_cache_type_t *type, ngx_str_t *id, uint32_t hash);
> +
> +
> +static ngx_core_module_t ngx_openssl_cache_module_ctx = {
> + ngx_string("openssl_cache"),
> + ngx_openssl_cache_create_conf,
> + NULL
> +};
> +
> +
> +ngx_module_t ngx_openssl_cache_module = {
> + NGX_MODULE_V1,
> + &ngx_openssl_cache_module_ctx, /* module context */
> + NULL, /* module directives */
> + NGX_CORE_MODULE, /* module type */
> + NULL, /* init master */
> + NULL, /* init module */
> + NULL, /* init process */
> + NULL, /* init thread */
> + NULL, /* exit thread */
> + NULL, /* exit process */
> + NULL, /* exit master */
> + NGX_MODULE_V1_PADDING
> +};
> +
> +
> +static void *
> +ngx_openssl_cache_create_conf(ngx_cycle_t *cycle)
> +{
> + ngx_ssl_cache_t *cache;
> + ngx_pool_cleanup_t *cln;
> +
> + cache = ngx_pcalloc(cycle->pool, sizeof(ngx_ssl_cache_t));
> + if (cache == NULL) {
> + return NULL;
> + }
> +
> + cln = ngx_pool_cleanup_add(cycle->pool, 0);
> + if (cln == NULL) {
> + return NULL;
> + }
> +
> + cln->handler = ngx_ssl_cache_cleanup;
> + cln->data = cache;
> +
> + ngx_rbtree_init(&cache->rbtree, &cache->sentinel,
> + ngx_ssl_cache_node_insert);
> +
> + return cache;
> +}
> +
> +
> +static void
> +ngx_ssl_cache_cleanup(void *data)
> +{
> + ngx_ssl_cache_t *cache = data;
> +
> + ngx_rbtree_t *tree;
> + ngx_rbtree_node_t *node;
> + ngx_ssl_cache_node_t *cn;
> +
> + tree = &cache->rbtree;
> +
> + if (tree->root == tree->sentinel) {
> + return;
> + }
> +
> + for (node = ngx_rbtree_min(tree->root, tree->sentinel);
> + node;
> + node = ngx_rbtree_next(tree, node))
> + {
> + cn = ngx_rbtree_data(node, ngx_ssl_cache_node_t, node);
> +
> + if (cn->type != NULL && cn->value != NULL) {
> + cn->type->free(cn->value);
> + }
> + }
> +}
> +
> +
> +static void
> +ngx_ssl_cache_node_insert(ngx_rbtree_node_t *temp,
> + ngx_rbtree_node_t *node, ngx_rbtree_node_t *sentinel)
> +{
> + ngx_rbtree_node_t **p;
> + ngx_ssl_cache_node_t *n, *t;
> +
> + for ( ;; ) {
> +
> + n = ngx_rbtree_data(node, ngx_ssl_cache_node_t, node);
> + t = ngx_rbtree_data(temp, ngx_ssl_cache_node_t, node);
> +
> + if (node->key != temp->key) {
> +
> + p = (node->key < temp->key) ? &temp->left : &temp->right;
> +
> + } else if (n->type != t->type) {
> +
> + p = (n->type < t->type) ? &temp->left : &temp->right;
> +
> + } else {
> +
> + p = (ngx_memn2cmp(n->id.data, t->id.data, n->id.len, t->id.len)
> + < 0) ? &temp->left : &temp->right;
> + }
> +
> + if (*p == sentinel) {
> + break;
> + }
> +
> + temp = *p;
> + }
> +
> + *p = node;
> + node->parent = temp;
> + node->left = sentinel;
> + node->right = sentinel;
> + ngx_rbt_red(node);
> +}
> +
> +
> +void *
> +ngx_ssl_cache_fetch(ngx_cycle_t *cycle, ngx_pool_t *pool,
> + ngx_ssl_cache_type_t *type, char **err, ngx_str_t *id, void *data)
> +{
> + void *value;
> + uint32_t hash;
> + ngx_ssl_cache_t *cache;
> + ngx_ssl_cache_node_t *cn;
> +
> + value = NULL;
> +
> + hash = ngx_murmur_hash2(id->data, id->len);
> +
> + cache = (ngx_ssl_cache_t *) ngx_get_conf(cycle->conf_ctx,
> + ngx_openssl_cache_module);
> +
> + if (ngx_process == NGX_PROCESS_WORKER
> + || ngx_process == NGX_PROCESS_SINGLE)
> + {
> + return type->create(id, err, data);
> + }
This check kept bothering me, so I figured we can try to improve it.
The logic for configuration time and per-connection fetches is quite
different and we're always aware of the context we're working in. It
would be cleaner to have separate fetch implementations, like that:
void *
ngx_ssl_cache_fetch(ngx_conf_t *cf, ngx_ssl_cache_type_t *type,
char **err,
ngx_str_t *id, void *data)
{
...
}
void *
ngx_ssl_cache_connection_fetch(ngx_ssl_cache_type_t *type, char **err,
ngx_str_t *id, void *data)
{
return type->create(id, err, data);
}
Full diff is attached, but it needs to be properly split between patches
2-6.
As a bonus, the change makes it easier to isolate any future logic for
dynamic certificate caching. It was noticeable on our review of early
prototypes how combining both code paths in the `ngx_ssl_cache_fetch`
code made the function unnecessarily complicated.
> +
> + cn = ngx_ssl_cache_lookup(cache, type, id, hash);
> +
> + if (cn == NULL) {
> + cn = ngx_palloc(pool, sizeof(ngx_ssl_cache_node_t) + id->len + 1);
> + if (cn == NULL) {
> + return NULL;
> + }
> +
> + cn->node.key = hash;
> + cn->id.data = (u_char *)(cn + 1);
> + cn->id.len = id->len;
> + cn->type = type;
> + cn->value = NULL;
> +
> + ngx_cpystrn(cn->id.data, id->data, id->len + 1);
> +
> + ngx_rbtree_insert(&cache->rbtree, &cn->node);
> + }
> +
> + /* try to use a reference from the cache */
> +
> + if (cn->value != NULL) {
> + value = type->ref(err, cn->value);
> + }
> +
> + if (value == NULL) {
> + value = type->create(id, err, data);
> + }
> +
> + if (value != NULL && cn->value == NULL) {
> + /* we have a value and the node needs one; try to reference it */
> + cn->value = type->ref(err, value);
> + }
> +
> + return value;
> +}
> +
> +
> +static ngx_ssl_cache_node_t *
> +ngx_ssl_cache_lookup(ngx_ssl_cache_t *cache, ngx_ssl_cache_type_t *type,
> + ngx_str_t *id, uint32_t hash)
> +{
> + ngx_int_t rc;
> + ngx_rbtree_node_t *node, *sentinel;
> + ngx_ssl_cache_node_t *cn;
> +
> + node = cache->rbtree.root;
> + sentinel = cache->rbtree.sentinel;
> +
> + while (node != sentinel) {
> +
> + if (hash < node->key) {
> + node = node->left;
> + continue;
> + }
> +
> + if (hash > node->key) {
> + node = node->right;
> + continue;
> + }
> +
> + /* hash == node->key */
> +
> + cn = (ngx_ssl_cache_node_t *) node;
> +
> + if ((ngx_uint_t) type < (ngx_uint_t) cn->type) {
> + node = node->left;
> + continue;
> + }
> +
> + if ((ngx_uint_t) type > (ngx_uint_t) cn->type) {
> + node = node->right;
> + continue;
> + }
> +
> + /* type == cn->type */
> +
> + rc = ngx_memn2cmp(id->data, cn->id.data, id->len, cn->id.len);
> +
> + if (rc == 0) {
> + return cn;
> + }
> +
> + node = (rc < 0) ? node->left : node->right;
> + }
> +
> + return NULL;
> +}
> _______________________________________________
> nginx-devel mailing list
> nginx-devel at nginx.org
> https://mailman.nginx.org/mailman/listinfo/nginx-devel
-------------- next part --------------
diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c
index e3f9c7bcc..ed8c8feff 100644
--- a/src/event/ngx_event_openssl.c
+++ b/src/event/ngx_event_openssl.c
@@ -439,8 +439,7 @@ ngx_ssl_certificate(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *cert,
STACK_OF(X509) *chain;
ngx_ssl_name_t *name;
- chain = ngx_ssl_cache_fetch(cf->cycle, cf->pool, &ngx_ssl_cache_cert,
- &err, cert, NULL);
+ chain = ngx_ssl_cache_fetch(cf, &ngx_ssl_cache_cert, &err, cert, NULL);
if (chain == NULL) {
if (err != NULL) {
ngx_ssl_error(NGX_LOG_EMERG, ssl->log, 0,
@@ -534,8 +533,7 @@ ngx_ssl_certificate(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *cert,
}
#endif
- pkey = ngx_ssl_cache_fetch(cf->cycle, cf->pool, &ngx_ssl_cache_key, &err,
- key, passwords);
+ pkey = ngx_ssl_cache_fetch(cf, &ngx_ssl_cache_key, &err, key, passwords);
if (pkey == NULL) {
if (err != NULL) {
ngx_ssl_error(NGX_LOG_EMERG, ssl->log, 0,
@@ -568,8 +566,8 @@ ngx_ssl_connection_certificate(ngx_connection_t *c, ngx_pool_t *pool,
EVP_PKEY *pkey;
STACK_OF(X509) *chain;
- chain = ngx_ssl_cache_fetch((ngx_cycle_t *) ngx_cycle, c->pool,
- &ngx_ssl_cache_cert, &err, cert, NULL);
+ chain = ngx_ssl_cache_connection_fetch(&ngx_ssl_cache_cert, &err, cert,
+ NULL);
if (chain == NULL) {
if (err != NULL) {
ngx_ssl_error(NGX_LOG_ERR, c->log, 0,
@@ -609,8 +607,8 @@ ngx_ssl_connection_certificate(ngx_connection_t *c, ngx_pool_t *pool,
#endif
- pkey = ngx_ssl_cache_fetch((ngx_cycle_t *) ngx_cycle, c->pool,
- &ngx_ssl_cache_key, &err, key, passwords);
+ pkey = ngx_ssl_cache_connection_fetch(&ngx_ssl_cache_key, &err, key,
+ passwords);
if (pkey == NULL) {
if (err != NULL) {
ngx_ssl_error(NGX_LOG_EMERG, c->log, 0,
@@ -686,8 +684,7 @@ ngx_ssl_client_certificate(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *cert,
}
xs = SSL_CTX_get_cert_store(ssl->ctx);
- xsk = ngx_ssl_cache_fetch(cf->cycle, cf->pool, &ngx_ssl_cache_ca, &err,
- cert, NULL);
+ xsk = ngx_ssl_cache_fetch(cf, &ngx_ssl_cache_ca, &err, cert, NULL);
if (xsk == NULL) {
if (err != NULL) {
ngx_ssl_error(NGX_LOG_EMERG, ssl->log, 0,
@@ -792,8 +789,7 @@ ngx_ssl_trusted_certificate(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *cert,
}
xs = SSL_CTX_get_cert_store(ssl->ctx);
- xsk = ngx_ssl_cache_fetch(cf->cycle, cf->pool, &ngx_ssl_cache_ca, &err,
- cert, NULL);
+ xsk = ngx_ssl_cache_fetch(cf, &ngx_ssl_cache_ca, &err, cert, NULL);
if (xsk == NULL) {
if (err != NULL) {
ngx_ssl_error(NGX_LOG_EMERG, ssl->log, 0,
@@ -860,8 +856,7 @@ ngx_ssl_crl(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *crl)
return NGX_ERROR;
}
- xcsk = ngx_ssl_cache_fetch(cf->cycle, cf->pool, &ngx_ssl_cache_crl,
- &err, crl, NULL);
+ xcsk = ngx_ssl_cache_fetch(cf, &ngx_ssl_cache_crl, &err, crl, NULL);
if (xcsk == NULL) {
if (err != NULL) {
ngx_ssl_error(NGX_LOG_EMERG, ssl->log, 0,
diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h
index 490f995b9..7bf79a017 100644
--- a/src/event/ngx_event_openssl.h
+++ b/src/event/ngx_event_openssl.h
@@ -234,8 +234,10 @@ ngx_int_t ngx_ssl_ocsp_get_status(ngx_connection_t *c, const char **s);
void ngx_ssl_ocsp_cleanup(ngx_connection_t *c);
ngx_int_t ngx_ssl_ocsp_cache_init(ngx_shm_zone_t *shm_zone, void *data);
-void *ngx_ssl_cache_fetch(ngx_cycle_t *cycle, ngx_pool_t *pool,
- ngx_ssl_cache_type_t *type, char **err, ngx_str_t *id, void *data);
+void *ngx_ssl_cache_fetch(ngx_conf_t *cf, ngx_ssl_cache_type_t *type,
+ char **err, ngx_str_t *id, void *data);
+void *ngx_ssl_cache_connection_fetch(ngx_ssl_cache_type_t *type, char **err,
+ ngx_str_t *id, void *data);
ngx_array_t *ngx_ssl_read_password_file(ngx_conf_t *cf, ngx_str_t *file);
ngx_array_t *ngx_ssl_preserve_passwords(ngx_conf_t *cf,
diff --git a/src/event/ngx_event_openssl_cache.c b/src/event/ngx_event_openssl_cache.c
index 7e207c916..64eae9c9c 100644
--- a/src/event/ngx_event_openssl_cache.c
+++ b/src/event/ngx_event_openssl_cache.c
@@ -219,8 +219,8 @@ ngx_ssl_cache_node_insert(ngx_rbtree_node_t *temp,
void *
-ngx_ssl_cache_fetch(ngx_cycle_t *cycle, ngx_pool_t *pool,
- ngx_ssl_cache_type_t *type, char **err, ngx_str_t *id, void *data)
+ngx_ssl_cache_fetch(ngx_conf_t *cf, ngx_ssl_cache_type_t *type, char **err,
+ ngx_str_t *id, void *data)
{
void *value;
uint32_t hash;
@@ -231,19 +231,13 @@ ngx_ssl_cache_fetch(ngx_cycle_t *cycle, ngx_pool_t *pool,
hash = ngx_murmur_hash2(id->data, id->len);
- cache = (ngx_ssl_cache_t *) ngx_get_conf(cycle->conf_ctx,
+ cache = (ngx_ssl_cache_t *) ngx_get_conf(cf->cycle->conf_ctx,
ngx_openssl_cache_module);
- if (ngx_process == NGX_PROCESS_WORKER
- || ngx_process == NGX_PROCESS_SINGLE)
- {
- return type->create(id, err, data);
- }
-
cn = ngx_ssl_cache_lookup(cache, type, id, hash);
if (cn == NULL) {
- cn = ngx_palloc(pool, sizeof(ngx_ssl_cache_node_t) + id->len + 1);
+ cn = ngx_palloc(cf->pool, sizeof(ngx_ssl_cache_node_t) + id->len + 1);
if (cn == NULL) {
return NULL;
}
@@ -278,6 +272,14 @@ ngx_ssl_cache_fetch(ngx_cycle_t *cycle, ngx_pool_t *pool,
}
+void *
+ngx_ssl_cache_connection_fetch(ngx_ssl_cache_type_t *type, char **err,
+ ngx_str_t *id, void *data)
+{
+ return type->create(id, err, data);
+}
+
+
static ngx_ssl_cache_node_t *
ngx_ssl_cache_lookup(ngx_ssl_cache_t *cache, ngx_ssl_cache_type_t *type,
ngx_str_t *id, uint32_t hash)
More information about the nginx-devel
mailing list