From fddcc30e999e56a9838a5ed818469afcd244f395 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Mon, 29 May 2023 15:03:31 +0400 Subject: [PATCH 001/279] Version bump. --- src/core/nginx.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/nginx.h b/src/core/nginx.h index 5bcca59f65b..7a32ef19124 100644 --- a/src/core/nginx.h +++ b/src/core/nginx.h @@ -9,8 +9,8 @@ #define _NGINX_H_INCLUDED_ -#define nginx_version 1025000 -#define NGINX_VERSION "1.25.0" +#define nginx_version 1025001 +#define NGINX_VERSION "1.25.1" #define NGINX_VER "nginx/" NGINX_VERSION #ifdef NGX_BUILD From cb70d5954c65b5683bc1c104bbf2466b73f4aa2b Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Sun, 28 May 2023 11:17:07 +0400 Subject: [PATCH 002/279] QUIC: fixed compat with ciphers other than AES128 (ticket #2500). Previously, rec.level field was not uninitialized in SSL_provide_quic_data(). As a result, its value was always ssl_encryption_initial. Later in ngx_quic_ciphers() such level resulted in resetting the cipher to TLS1_3_CK_AES_128_GCM_SHA256 and using AES128 to encrypt the packet. Now the level is initialized and the right cipher is used. --- src/event/quic/ngx_event_quic_openssl_compat.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/event/quic/ngx_event_quic_openssl_compat.c b/src/event/quic/ngx_event_quic_openssl_compat.c index 51430e4b972..63d380e35bd 100644 --- a/src/event/quic/ngx_event_quic_openssl_compat.c +++ b/src/event/quic/ngx_event_quic_openssl_compat.c @@ -463,6 +463,7 @@ SSL_provide_quic_data(SSL *ssl, enum ssl_encryption_level_t level, rec.log = c->log; rec.number = com->read_record++; rec.keys = &com->keys; + rec.level = level; if (level == ssl_encryption_initial) { n = ngx_min(len, 65535); From aefd862ab197c3ab49001fcf69be478aab5b0f4e Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Tue, 16 May 2023 16:30:08 +0400 Subject: [PATCH 003/279] HTTP/2: "http2" directive. The directive enables HTTP/2 in the current server. The previous way to enable HTTP/2 via "listen ... http2" is now deprecated. The new approach allows to share HTTP/2 and HTTP/0.9-1.1 on the same port. For SSL connections, HTTP/2 is now selected by ALPN callback based on whether the protocol is enabled in the virtual server chosen by SNI. This however only works since OpenSSL 1.0.2h, where ALPN callback is invoked after SNI callback. For older versions of OpenSSL, HTTP/2 is enabled based on the default virtual server configuration. For plain TCP connections, HTTP/2 is now auto-detected by HTTP/2 preface, if HTTP/2 is enabled in the default virtual server. If preface is not matched, HTTP/0.9-1.1 is assumed. --- src/http/modules/ngx_http_ssl_module.c | 24 +++++++---- src/http/ngx_http_core_module.c | 5 +++ src/http/ngx_http_request.c | 59 ++++++++++++++++++------- src/http/v2/ngx_http_v2.c | 60 ++++++++++++++------------ src/http/v2/ngx_http_v2.h | 18 ++++++++ src/http/v2/ngx_http_v2_module.c | 11 +++++ src/http/v2/ngx_http_v2_module.h | 12 ------ 7 files changed, 127 insertions(+), 62 deletions(-) diff --git a/src/http/modules/ngx_http_ssl_module.c b/src/http/modules/ngx_http_ssl_module.c index d2ca475d325..3d52c1c57ad 100644 --- a/src/http/modules/ngx_http_ssl_module.c +++ b/src/http/modules/ngx_http_ssl_module.c @@ -435,6 +435,9 @@ ngx_http_ssl_alpn_select(ngx_ssl_conn_t *ssl_conn, const unsigned char **out, #if (NGX_HTTP_V2 || NGX_HTTP_V3) ngx_http_connection_t *hc; #endif +#if (NGX_HTTP_V2) + ngx_http_v2_srv_conf_t *h2scf; +#endif #if (NGX_HTTP_V3) ngx_http_v3_srv_conf_t *h3scf; #endif @@ -456,12 +459,6 @@ ngx_http_ssl_alpn_select(ngx_ssl_conn_t *ssl_conn, const unsigned char **out, hc = c->data; #endif -#if (NGX_HTTP_V2) - if (hc->addr_conf->http2) { - srv = (unsigned char *) NGX_HTTP_V2_ALPN_PROTO NGX_HTTP_ALPN_PROTOS; - srvlen = sizeof(NGX_HTTP_V2_ALPN_PROTO NGX_HTTP_ALPN_PROTOS) - 1; - } else -#endif #if (NGX_HTTP_V3) if (hc->addr_conf->quic) { @@ -488,8 +485,19 @@ ngx_http_ssl_alpn_select(ngx_ssl_conn_t *ssl_conn, const unsigned char **out, } else #endif { - srv = (unsigned char *) NGX_HTTP_ALPN_PROTOS; - srvlen = sizeof(NGX_HTTP_ALPN_PROTOS) - 1; +#if (NGX_HTTP_V2) + h2scf = ngx_http_get_module_srv_conf(hc->conf_ctx, ngx_http_v2_module); + + if (h2scf->enable || hc->addr_conf->http2) { + srv = (unsigned char *) NGX_HTTP_V2_ALPN_PROTO NGX_HTTP_ALPN_PROTOS; + srvlen = sizeof(NGX_HTTP_V2_ALPN_PROTO NGX_HTTP_ALPN_PROTOS) - 1; + + } else +#endif + { + srv = (unsigned char *) NGX_HTTP_ALPN_PROTOS; + srvlen = sizeof(NGX_HTTP_ALPN_PROTOS) - 1; + } } if (SSL_select_next_proto((unsigned char **) out, outlen, srv, srvlen, diff --git a/src/http/ngx_http_core_module.c b/src/http/ngx_http_core_module.c index bd8f7666a39..97a91aee266 100644 --- a/src/http/ngx_http_core_module.c +++ b/src/http/ngx_http_core_module.c @@ -4176,6 +4176,11 @@ ngx_http_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) if (ngx_strcmp(value[n].data, "http2") == 0) { #if (NGX_HTTP_V2) + ngx_conf_log_error(NGX_LOG_WARN, cf, 0, + "the \"listen ... http2\" directive " + "is deprecated, use " + "the \"http2\" directive instead"); + lsopt.http2 = 1; continue; #else diff --git a/src/http/ngx_http_request.c b/src/http/ngx_http_request.c index 1f09c724eb2..0aca921329c 100644 --- a/src/http/ngx_http_request.c +++ b/src/http/ngx_http_request.c @@ -318,12 +318,6 @@ ngx_http_init_connection(ngx_connection_t *c) rev->handler = ngx_http_wait_request_handler; c->write->handler = ngx_http_empty_handler; -#if (NGX_HTTP_V2) - if (hc->addr_conf->http2) { - rev->handler = ngx_http_v2_init; - } -#endif - #if (NGX_HTTP_V3) if (hc->addr_conf->quic) { ngx_http_v3_init_stream(c); @@ -383,6 +377,9 @@ ngx_http_wait_request_handler(ngx_event_t *rev) ngx_buf_t *b; ngx_connection_t *c; ngx_http_connection_t *hc; +#if (NGX_HTTP_V2) + ngx_http_v2_srv_conf_t *h2scf; +#endif ngx_http_core_srv_conf_t *cscf; c = rev->data; @@ -429,6 +426,8 @@ ngx_http_wait_request_handler(ngx_event_t *rev) b->end = b->last + size; } + size = b->end - b->last; + n = c->recv(c, b->last, size); if (n == NGX_AGAIN) { @@ -443,12 +442,16 @@ ngx_http_wait_request_handler(ngx_event_t *rev) return; } - /* - * We are trying to not hold c->buffer's memory for an idle connection. - */ + if (b->pos == b->last) { - if (ngx_pfree(c->pool, b->start) == NGX_OK) { - b->start = NULL; + /* + * We are trying to not hold c->buffer's memory for an + * idle connection. + */ + + if (ngx_pfree(c->pool, b->start) == NGX_OK) { + b->start = NULL; + } } return; @@ -489,6 +492,29 @@ ngx_http_wait_request_handler(ngx_event_t *rev) } } +#if (NGX_HTTP_V2) + + h2scf = ngx_http_get_module_srv_conf(hc->conf_ctx, ngx_http_v2_module); + + if (!hc->ssl && (h2scf->enable || hc->addr_conf->http2)) { + + size = ngx_min(sizeof(NGX_HTTP_V2_PREFACE) - 1, + (size_t) (b->last - b->pos)); + + if (ngx_memcmp(b->pos, NGX_HTTP_V2_PREFACE, size) == 0) { + + if (size == sizeof(NGX_HTTP_V2_PREFACE) - 1) { + ngx_http_v2_init(rev); + return; + } + + ngx_post_event(rev, &ngx_posted_events); + return; + } + } + +#endif + c->log->action = "reading client request line"; ngx_reusable_connection(c, 0); @@ -808,13 +834,16 @@ ngx_http_ssl_handshake_handler(ngx_connection_t *c) #if (NGX_HTTP_V2 \ && defined TLSEXT_TYPE_application_layer_protocol_negotiation) { - unsigned int len; - const unsigned char *data; - ngx_http_connection_t *hc; + unsigned int len; + const unsigned char *data; + ngx_http_connection_t *hc; + ngx_http_v2_srv_conf_t *h2scf; hc = c->data; - if (hc->addr_conf->http2) { + h2scf = ngx_http_get_module_srv_conf(hc->conf_ctx, ngx_http_v2_module); + + if (h2scf->enable || hc->addr_conf->http2) { SSL_get0_alpn_selected(c->ssl->connection, &data, &len); diff --git a/src/http/v2/ngx_http_v2.c b/src/http/v2/ngx_http_v2.c index ea3f27c0785..deb2bf1ba26 100644 --- a/src/http/v2/ngx_http_v2.c +++ b/src/http/v2/ngx_http_v2.c @@ -63,8 +63,6 @@ static void ngx_http_v2_handle_connection(ngx_http_v2_connection_t *h2c); static void ngx_http_v2_lingering_close(ngx_connection_t *c); static void ngx_http_v2_lingering_close_handler(ngx_event_t *rev); -static u_char *ngx_http_v2_state_proxy_protocol(ngx_http_v2_connection_t *h2c, - u_char *pos, u_char *end); static u_char *ngx_http_v2_state_preface(ngx_http_v2_connection_t *h2c, u_char *pos, u_char *end); static u_char *ngx_http_v2_state_preface_end(ngx_http_v2_connection_t *h2c, @@ -232,6 +230,7 @@ static ngx_http_v2_parse_header_t ngx_http_v2_parse_headers[] = { void ngx_http_v2_init(ngx_event_t *rev) { + u_char *p, *end; ngx_connection_t *c; ngx_pool_cleanup_t *cln; ngx_http_connection_t *hc; @@ -314,8 +313,7 @@ ngx_http_v2_init(ngx_event_t *rev) return; } - h2c->state.handler = hc->proxy_protocol ? ngx_http_v2_state_proxy_protocol - : ngx_http_v2_state_preface; + h2c->state.handler = ngx_http_v2_state_preface; ngx_queue_init(&h2c->waiting); ngx_queue_init(&h2c->dependencies); @@ -335,6 +333,23 @@ ngx_http_v2_init(ngx_event_t *rev) c->idle = 1; ngx_reusable_connection(c, 0); + if (c->buffer) { + p = c->buffer->pos; + end = c->buffer->last; + + do { + p = h2c->state.handler(h2c, p, end); + + if (p == NULL) { + return; + } + + } while (p != end); + + h2c->total_bytes += p - c->buffer->pos; + c->buffer->pos = p; + } + ngx_http_v2_read_handler(rev); } @@ -846,32 +861,11 @@ ngx_http_v2_lingering_close_handler(ngx_event_t *rev) } -static u_char * -ngx_http_v2_state_proxy_protocol(ngx_http_v2_connection_t *h2c, u_char *pos, - u_char *end) -{ - ngx_log_t *log; - - log = h2c->connection->log; - log->action = "reading PROXY protocol"; - - pos = ngx_proxy_protocol_read(h2c->connection, pos, end); - - log->action = "processing HTTP/2 connection"; - - if (pos == NULL) { - return ngx_http_v2_connection_error(h2c, NGX_HTTP_V2_PROTOCOL_ERROR); - } - - return ngx_http_v2_state_preface(h2c, pos, end); -} - - static u_char * ngx_http_v2_state_preface(ngx_http_v2_connection_t *h2c, u_char *pos, u_char *end) { - static const u_char preface[] = "PRI * HTTP/2.0\r\n"; + static const u_char preface[] = NGX_HTTP_V2_PREFACE_START; if ((size_t) (end - pos) < sizeof(preface) - 1) { return ngx_http_v2_state_save(h2c, pos, end, ngx_http_v2_state_preface); @@ -892,7 +886,7 @@ static u_char * ngx_http_v2_state_preface_end(ngx_http_v2_connection_t *h2c, u_char *pos, u_char *end) { - static const u_char preface[] = "\r\nSM\r\n\r\n"; + static const u_char preface[] = NGX_HTTP_V2_PREFACE_END; if ((size_t) (end - pos) < sizeof(preface) - 1) { return ngx_http_v2_state_save(h2c, pos, end, @@ -3943,10 +3937,22 @@ static void ngx_http_v2_run_request(ngx_http_request_t *r) { ngx_connection_t *fc; + ngx_http_v2_srv_conf_t *h2scf; ngx_http_v2_connection_t *h2c; fc = r->connection; + h2scf = ngx_http_get_module_srv_conf(r, ngx_http_v2_module); + + if (!h2scf->enable && !r->http_connection->addr_conf->http2) { + ngx_log_error(NGX_LOG_INFO, fc->log, 0, + "client attempted to request the server name " + "for which the negotiated protocol is disabled"); + + ngx_http_finalize_request(r, NGX_HTTP_MISDIRECTED_REQUEST); + goto failed; + } + if (ngx_http_v2_construct_request_line(r) != NGX_OK) { goto failed; } diff --git a/src/http/v2/ngx_http_v2.h b/src/http/v2/ngx_http_v2.h index 4e252931ce6..cb7a313e01b 100644 --- a/src/http/v2/ngx_http_v2.h +++ b/src/http/v2/ngx_http_v2.h @@ -63,6 +63,16 @@ typedef u_char *(*ngx_http_v2_handler_pt) (ngx_http_v2_connection_t *h2c, u_char *pos, u_char *end); +typedef struct { + ngx_flag_t enable; + size_t pool_size; + ngx_uint_t concurrent_streams; + ngx_uint_t concurrent_pushes; + size_t preread_size; + ngx_uint_t streams_index_mask; +} ngx_http_v2_srv_conf_t; + + typedef struct { ngx_str_t name; ngx_str_t value; @@ -408,9 +418,17 @@ ngx_int_t ngx_http_v2_table_size(ngx_http_v2_connection_t *h2c, size_t size); #define NGX_HTTP_V2_USER_AGENT_INDEX 58 #define NGX_HTTP_V2_VARY_INDEX 59 +#define NGX_HTTP_V2_PREFACE_START "PRI * HTTP/2.0\r\n" +#define NGX_HTTP_V2_PREFACE_END "\r\nSM\r\n\r\n" +#define NGX_HTTP_V2_PREFACE NGX_HTTP_V2_PREFACE_START \ + NGX_HTTP_V2_PREFACE_END + u_char *ngx_http_v2_string_encode(u_char *dst, u_char *src, size_t len, u_char *tmp, ngx_uint_t lower); +extern ngx_module_t ngx_http_v2_module; + + #endif /* _NGX_HTTP_V2_H_INCLUDED_ */ diff --git a/src/http/v2/ngx_http_v2_module.c b/src/http/v2/ngx_http_v2_module.c index 00508861118..09396a50b6f 100644 --- a/src/http/v2/ngx_http_v2_module.c +++ b/src/http/v2/ngx_http_v2_module.c @@ -75,6 +75,13 @@ static ngx_conf_post_t ngx_http_v2_chunk_size_post = static ngx_command_t ngx_http_v2_commands[] = { + { ngx_string("http2"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, + ngx_conf_set_flag_slot, + NGX_HTTP_SRV_CONF_OFFSET, + offsetof(ngx_http_v2_srv_conf_t, enable), + NULL }, + { ngx_string("http2_recv_buffer_size"), NGX_HTTP_MAIN_CONF|NGX_CONF_TAKE1, ngx_conf_set_size_slot, @@ -314,6 +321,8 @@ ngx_http_v2_create_srv_conf(ngx_conf_t *cf) return NULL; } + h2scf->enable = NGX_CONF_UNSET; + h2scf->pool_size = NGX_CONF_UNSET_SIZE; h2scf->concurrent_streams = NGX_CONF_UNSET_UINT; @@ -333,6 +342,8 @@ ngx_http_v2_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) ngx_http_v2_srv_conf_t *prev = parent; ngx_http_v2_srv_conf_t *conf = child; + ngx_conf_merge_value(conf->enable, prev->enable, 0); + ngx_conf_merge_size_value(conf->pool_size, prev->pool_size, 4096); ngx_conf_merge_uint_value(conf->concurrent_streams, diff --git a/src/http/v2/ngx_http_v2_module.h b/src/http/v2/ngx_http_v2_module.h index ca4a0bfc549..22a2d84ca09 100644 --- a/src/http/v2/ngx_http_v2_module.h +++ b/src/http/v2/ngx_http_v2_module.h @@ -20,15 +20,6 @@ typedef struct { } ngx_http_v2_main_conf_t; -typedef struct { - size_t pool_size; - ngx_uint_t concurrent_streams; - ngx_uint_t concurrent_pushes; - size_t preread_size; - ngx_uint_t streams_index_mask; -} ngx_http_v2_srv_conf_t; - - typedef struct { size_t chunk_size; @@ -39,7 +30,4 @@ typedef struct { } ngx_http_v2_loc_conf_t; -extern ngx_module_t ngx_http_v2_module; - - #endif /* _NGX_HTTP_V2_MODULE_H_INCLUDED_ */ From d32f66f1e8dc81a0edbadbacf74191684a653d09 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Thu, 8 Jun 2023 14:49:27 +0400 Subject: [PATCH 004/279] SSL: removed the "ssl" directive. It has been deprecated since 7270:46c0c7ef4913 (1.15.0) in favour of the "ssl" parameter of the "listen" directive, which has been available since 2224:109849282793 (0.7.14). --- src/http/modules/ngx_http_ssl_module.c | 78 +------------------------- src/http/modules/ngx_http_ssl_module.h | 5 -- src/http/ngx_http_request.c | 8 +-- src/mail/ngx_mail_handler.c | 6 +- src/mail/ngx_mail_ssl_module.c | 53 ----------------- src/mail/ngx_mail_ssl_module.h | 1 - 6 files changed, 5 insertions(+), 146 deletions(-) diff --git a/src/http/modules/ngx_http_ssl_module.c b/src/http/modules/ngx_http_ssl_module.c index 3d52c1c57ad..1c92d9fa879 100644 --- a/src/http/modules/ngx_http_ssl_module.c +++ b/src/http/modules/ngx_http_ssl_module.c @@ -43,8 +43,6 @@ static char *ngx_http_ssl_merge_srv_conf(ngx_conf_t *cf, static ngx_int_t ngx_http_ssl_compile_certificates(ngx_conf_t *cf, ngx_http_ssl_srv_conf_t *conf); -static char *ngx_http_ssl_enable(ngx_conf_t *cf, ngx_command_t *cmd, - void *conf); static char *ngx_http_ssl_password_file(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); static char *ngx_http_ssl_session_cache(ngx_conf_t *cf, ngx_command_t *cmd, @@ -90,24 +88,12 @@ static ngx_conf_enum_t ngx_http_ssl_ocsp[] = { }; -static ngx_conf_deprecated_t ngx_http_ssl_deprecated = { - ngx_conf_deprecated, "ssl", "listen ... ssl" -}; - - static ngx_conf_post_t ngx_http_ssl_conf_command_post = { ngx_http_ssl_conf_command_check }; static ngx_command_t ngx_http_ssl_commands[] = { - { ngx_string("ssl"), - NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, - ngx_http_ssl_enable, - NGX_HTTP_SRV_CONF_OFFSET, - offsetof(ngx_http_ssl_srv_conf_t, enable), - &ngx_http_ssl_deprecated }, - { ngx_string("ssl_certificate"), NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1, ngx_conf_set_str_array_slot, @@ -625,7 +611,6 @@ ngx_http_ssl_create_srv_conf(ngx_conf_t *cf) * sscf->stapling_responder = { 0, NULL }; */ - sscf->enable = NGX_CONF_UNSET; sscf->prefer_server_ciphers = NGX_CONF_UNSET; sscf->early_data = NGX_CONF_UNSET; sscf->reject_handshake = NGX_CONF_UNSET; @@ -657,17 +642,6 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) ngx_pool_cleanup_t *cln; - if (conf->enable == NGX_CONF_UNSET) { - if (prev->enable == NGX_CONF_UNSET) { - conf->enable = 0; - - } else { - conf->enable = prev->enable; - conf->file = prev->file; - conf->line = prev->line; - } - } - ngx_conf_merge_value(conf->session_timeout, prev->session_timeout, 300); @@ -722,37 +696,7 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) conf->ssl.log = cf->log; - if (conf->enable) { - - if (conf->certificates) { - if (conf->certificate_keys == NULL) { - ngx_log_error(NGX_LOG_EMERG, cf->log, 0, - "no \"ssl_certificate_key\" is defined for " - "the \"ssl\" directive in %s:%ui", - conf->file, conf->line); - return NGX_CONF_ERROR; - } - - if (conf->certificate_keys->nelts < conf->certificates->nelts) { - ngx_log_error(NGX_LOG_EMERG, cf->log, 0, - "no \"ssl_certificate_key\" is defined " - "for certificate \"%V\" and " - "the \"ssl\" directive in %s:%ui", - ((ngx_str_t *) conf->certificates->elts) - + conf->certificates->nelts - 1, - conf->file, conf->line); - return NGX_CONF_ERROR; - } - - } else if (!conf->reject_handshake) { - ngx_log_error(NGX_LOG_EMERG, cf->log, 0, - "no \"ssl_certificate\" is defined for " - "the \"ssl\" directive in %s:%ui", - conf->file, conf->line); - return NGX_CONF_ERROR; - } - - } else if (conf->certificates) { + if (conf->certificates) { if (conf->certificate_keys == NULL || conf->certificate_keys->nelts < conf->certificates->nelts) @@ -1038,26 +982,6 @@ ngx_http_ssl_compile_certificates(ngx_conf_t *cf, } -static char * -ngx_http_ssl_enable(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) -{ - ngx_http_ssl_srv_conf_t *sscf = conf; - - char *rv; - - rv = ngx_conf_set_flag_slot(cf, cmd, conf); - - if (rv != NGX_CONF_OK) { - return rv; - } - - sscf->file = cf->conf_file->file.name.data; - sscf->line = cf->conf_file->line; - - return NGX_CONF_OK; -} - - static char * ngx_http_ssl_password_file(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) { diff --git a/src/http/modules/ngx_http_ssl_module.h b/src/http/modules/ngx_http_ssl_module.h index 7ab0f7eae28..c69c8ffd2bf 100644 --- a/src/http/modules/ngx_http_ssl_module.h +++ b/src/http/modules/ngx_http_ssl_module.h @@ -15,8 +15,6 @@ typedef struct { - ngx_flag_t enable; - ngx_ssl_t ssl; ngx_flag_t prefer_server_ciphers; @@ -64,9 +62,6 @@ typedef struct { ngx_flag_t stapling_verify; ngx_str_t stapling_file; ngx_str_t stapling_responder; - - u_char *file; - ngx_uint_t line; } ngx_http_ssl_srv_conf_t; diff --git a/src/http/ngx_http_request.c b/src/http/ngx_http_request.c index 0aca921329c..bd2be5eacd6 100644 --- a/src/http/ngx_http_request.c +++ b/src/http/ngx_http_request.c @@ -326,17 +326,11 @@ ngx_http_init_connection(ngx_connection_t *c) #endif #if (NGX_HTTP_SSL) - { - ngx_http_ssl_srv_conf_t *sscf; - - sscf = ngx_http_get_module_srv_conf(hc->conf_ctx, ngx_http_ssl_module); - - if (sscf->enable || hc->addr_conf->ssl) { + if (hc->addr_conf->ssl) { hc->ssl = 1; c->log->action = "SSL handshaking"; rev->handler = ngx_http_ssl_handshake; } - } #endif if (hc->addr_conf->proxy_protocol) { diff --git a/src/mail/ngx_mail_handler.c b/src/mail/ngx_mail_handler.c index 246ba97cf91..1167df3fb37 100644 --- a/src/mail/ngx_mail_handler.c +++ b/src/mail/ngx_mail_handler.c @@ -283,11 +283,11 @@ ngx_mail_init_session_handler(ngx_event_t *rev) s = c->data; - sslcf = ngx_mail_get_module_srv_conf(s, ngx_mail_ssl_module); - - if (sslcf->enable || s->ssl) { + if (s->ssl) { c->log->action = "SSL handshaking"; + sslcf = ngx_mail_get_module_srv_conf(s, ngx_mail_ssl_module); + ngx_mail_ssl_init_connection(&sslcf->ssl, c); return; } diff --git a/src/mail/ngx_mail_ssl_module.c b/src/mail/ngx_mail_ssl_module.c index 28737ac4e56..aebb4ccb6f8 100644 --- a/src/mail/ngx_mail_ssl_module.c +++ b/src/mail/ngx_mail_ssl_module.c @@ -23,8 +23,6 @@ static int ngx_mail_ssl_alpn_select(ngx_ssl_conn_t *ssl_conn, static void *ngx_mail_ssl_create_conf(ngx_conf_t *cf); static char *ngx_mail_ssl_merge_conf(ngx_conf_t *cf, void *parent, void *child); -static char *ngx_mail_ssl_enable(ngx_conf_t *cf, ngx_command_t *cmd, - void *conf); static char *ngx_mail_ssl_starttls(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); static char *ngx_mail_ssl_password_file(ngx_conf_t *cf, ngx_command_t *cmd, @@ -65,24 +63,12 @@ static ngx_conf_enum_t ngx_mail_ssl_verify[] = { }; -static ngx_conf_deprecated_t ngx_mail_ssl_deprecated = { - ngx_conf_deprecated, "ssl", "listen ... ssl" -}; - - static ngx_conf_post_t ngx_mail_ssl_conf_command_post = { ngx_mail_ssl_conf_command_check }; static ngx_command_t ngx_mail_ssl_commands[] = { - { ngx_string("ssl"), - NGX_MAIL_MAIN_CONF|NGX_MAIL_SRV_CONF|NGX_CONF_FLAG, - ngx_mail_ssl_enable, - NGX_MAIL_SRV_CONF_OFFSET, - offsetof(ngx_mail_ssl_conf_t, enable), - &ngx_mail_ssl_deprecated }, - { ngx_string("starttls"), NGX_MAIL_MAIN_CONF|NGX_MAIL_SRV_CONF|NGX_CONF_TAKE1, ngx_mail_ssl_starttls, @@ -322,7 +308,6 @@ ngx_mail_ssl_create_conf(ngx_conf_t *cf) * scf->shm_zone = NULL; */ - scf->enable = NGX_CONF_UNSET; scf->starttls = NGX_CONF_UNSET_UINT; scf->certificates = NGX_CONF_UNSET_PTR; scf->certificate_keys = NGX_CONF_UNSET_PTR; @@ -349,7 +334,6 @@ ngx_mail_ssl_merge_conf(ngx_conf_t *cf, void *parent, void *child) char *mode; ngx_pool_cleanup_t *cln; - ngx_conf_merge_value(conf->enable, prev->enable, 0); ngx_conf_merge_uint_value(conf->starttls, prev->starttls, NGX_MAIL_STARTTLS_OFF); @@ -394,9 +378,6 @@ ngx_mail_ssl_merge_conf(ngx_conf_t *cf, void *parent, void *child) if (conf->listen) { mode = "listen ... ssl"; - } else if (conf->enable) { - mode = "ssl"; - } else if (conf->starttls != NGX_MAIL_STARTTLS_OFF) { mode = "starttls"; @@ -545,34 +526,6 @@ ngx_mail_ssl_merge_conf(ngx_conf_t *cf, void *parent, void *child) } -static char * -ngx_mail_ssl_enable(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) -{ - ngx_mail_ssl_conf_t *scf = conf; - - char *rv; - - rv = ngx_conf_set_flag_slot(cf, cmd, conf); - - if (rv != NGX_CONF_OK) { - return rv; - } - - if (scf->enable && (ngx_int_t) scf->starttls > NGX_MAIL_STARTTLS_OFF) { - ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, - "\"starttls\" directive conflicts with \"ssl on\""); - return NGX_CONF_ERROR; - } - - if (!scf->listen) { - scf->file = cf->conf_file->file.name.data; - scf->line = cf->conf_file->line; - } - - return NGX_CONF_OK; -} - - static char * ngx_mail_ssl_starttls(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) { @@ -586,12 +539,6 @@ ngx_mail_ssl_starttls(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) return rv; } - if (scf->enable == 1 && (ngx_int_t) scf->starttls > NGX_MAIL_STARTTLS_OFF) { - ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, - "\"ssl\" directive conflicts with \"starttls\""); - return NGX_CONF_ERROR; - } - if (!scf->listen) { scf->file = cf->conf_file->file.name.data; scf->line = cf->conf_file->line; diff --git a/src/mail/ngx_mail_ssl_module.h b/src/mail/ngx_mail_ssl_module.h index a0a61131735..c0eb6a38f83 100644 --- a/src/mail/ngx_mail_ssl_module.h +++ b/src/mail/ngx_mail_ssl_module.h @@ -20,7 +20,6 @@ typedef struct { - ngx_flag_t enable; ngx_flag_t prefer_server_ciphers; ngx_ssl_t ssl; From 6915d2fb2e88e0c339fe37b37ce14f5fe446c1c6 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Thu, 8 Jun 2023 16:56:46 +0400 Subject: [PATCH 005/279] HTTP/2: removed server push (ticket #2432). Although it has better implementation status than HTTP/3 server push, it remains of limited use, with adoption numbers seen as negligible. Per IETF 102 materials, server push was used only in 0.04% of sessions. It was considered to be "difficult to use effectively" in RFC 9113. Its use is further limited by badly matching to fetch/cache/connection models in browsers, see related discussions linked from [1]. Server push was disabled in Chrome 106 [2]. The http2_push, http2_push_preload, and http2_max_concurrent_pushes directives are made obsolete. In particular, this essentially reverts 7201:641306096f5b and 7207:3d2b0b02bd3d. [1] https://jakearchibald.com/2017/h2-push-tougher-than-i-thought/ [2] https://chromestatus.com/feature/6302414934114304 --- auto/modules | 2 - src/http/v2/ngx_http_v2.c | 314 ++---------- src/http/v2/ngx_http_v2.h | 14 - src/http/v2/ngx_http_v2_filter_module.c | 622 +----------------------- src/http/v2/ngx_http_v2_module.c | 111 +---- src/http/v2/ngx_http_v2_module.h | 5 - 6 files changed, 56 insertions(+), 1012 deletions(-) diff --git a/auto/modules b/auto/modules index 76e6531c530..300d07cc226 100644 --- a/auto/modules +++ b/auto/modules @@ -423,7 +423,6 @@ if [ $HTTP = YES ]; then if [ $HTTP_V2 = YES ]; then have=NGX_HTTP_V2 . auto/have - have=NGX_HTTP_HEADERS . auto/have ngx_module_name=ngx_http_v2_module ngx_module_incs=src/http/v2 @@ -444,7 +443,6 @@ if [ $HTTP = YES ]; then HTTP_SSL=YES have=NGX_HTTP_V3 . auto/have - have=NGX_HTTP_HEADERS . auto/have ngx_module_name=ngx_http_v3_module ngx_module_incs=src/http/v3 diff --git a/src/http/v2/ngx_http_v2.c b/src/http/v2/ngx_http_v2.c index deb2bf1ba26..7c05ff1e78b 100644 --- a/src/http/v2/ngx_http_v2.c +++ b/src/http/v2/ngx_http_v2.c @@ -11,14 +11,6 @@ #include -typedef struct { - ngx_str_t name; - ngx_uint_t offset; - ngx_uint_t hash; - ngx_http_header_t *hh; -} ngx_http_v2_parse_header_t; - - /* errors */ #define NGX_HTTP_V2_NO_ERROR 0x0 #define NGX_HTTP_V2_PROTOCOL_ERROR 0x1 @@ -126,7 +118,7 @@ static ngx_int_t ngx_http_v2_parse_int(ngx_http_v2_connection_t *h2c, u_char **pos, u_char *end, ngx_uint_t prefix); static ngx_http_v2_stream_t *ngx_http_v2_create_stream( - ngx_http_v2_connection_t *h2c, ngx_uint_t push); + ngx_http_v2_connection_t *h2c); static ngx_http_v2_node_t *ngx_http_v2_get_node_by_id( ngx_http_v2_connection_t *h2c, ngx_uint_t sid, ngx_uint_t alloc); static ngx_http_v2_node_t *ngx_http_v2_get_closed_node( @@ -162,14 +154,11 @@ static ngx_int_t ngx_http_v2_parse_scheme(ngx_http_request_t *r, ngx_str_t *value); static ngx_int_t ngx_http_v2_parse_authority(ngx_http_request_t *r, ngx_str_t *value); -static ngx_int_t ngx_http_v2_parse_header(ngx_http_request_t *r, - ngx_http_v2_parse_header_t *header, ngx_str_t *value); static ngx_int_t ngx_http_v2_construct_request_line(ngx_http_request_t *r); static ngx_int_t ngx_http_v2_cookie(ngx_http_request_t *r, ngx_http_v2_header_t *header); static ngx_int_t ngx_http_v2_construct_cookie_header(ngx_http_request_t *r); static void ngx_http_v2_run_request(ngx_http_request_t *r); -static void ngx_http_v2_run_request_handler(ngx_event_t *ev); static ngx_int_t ngx_http_v2_process_request_body(ngx_http_request_t *r, u_char *pos, size_t size, ngx_uint_t last, ngx_uint_t flush); static ngx_int_t ngx_http_v2_filter_request_body(ngx_http_request_t *r); @@ -210,23 +199,6 @@ static ngx_http_v2_handler_pt ngx_http_v2_frame_states[] = { (sizeof(ngx_http_v2_frame_states) / sizeof(ngx_http_v2_handler_pt)) -static ngx_http_v2_parse_header_t ngx_http_v2_parse_headers[] = { - { ngx_string("host"), - offsetof(ngx_http_headers_in_t, host), 0, NULL }, - - { ngx_string("accept-encoding"), - offsetof(ngx_http_headers_in_t, accept_encoding), 0, NULL }, - - { ngx_string("accept-language"), - offsetof(ngx_http_headers_in_t, accept_language), 0, NULL }, - - { ngx_string("user-agent"), - offsetof(ngx_http_headers_in_t, user_agent), 0, NULL }, - - { ngx_null_string, 0, 0, NULL } -}; - - void ngx_http_v2_init(ngx_event_t *rev) { @@ -275,7 +247,6 @@ ngx_http_v2_init(ngx_event_t *rev) h2scf = ngx_http_get_module_srv_conf(hc->conf_ctx, ngx_http_v2_module); - h2c->concurrent_pushes = h2scf->concurrent_pushes; h2c->priority_limit = ngx_max(h2scf->concurrent_streams, 100); h2c->pool = ngx_create_pool(h2scf->pool_size, h2c->connection->log); @@ -385,7 +356,7 @@ ngx_http_v2_read_handler(ngx_event_t *rev) return; } - if (!h2c->processing && !h2c->pushing) { + if (!h2c->processing) { ngx_http_v2_finalize_connection(h2c, NGX_HTTP_V2_NO_ERROR); return; } @@ -428,9 +399,7 @@ ngx_http_v2_read_handler(ngx_event_t *rev) break; } - if (n == 0 - && (h2c->state.incomplete || h2c->processing || h2c->pushing)) - { + if (n == 0 && (h2c->state.incomplete || h2c->processing)) { ngx_log_error(NGX_LOG_INFO, c->log, 0, "client prematurely closed connection"); } @@ -653,7 +622,7 @@ ngx_http_v2_handle_connection(ngx_http_v2_connection_t *h2c) ngx_connection_t *c; ngx_http_core_loc_conf_t *clcf; - if (h2c->last_out || h2c->processing || h2c->pushing) { + if (h2c->last_out || h2c->processing) { return; } @@ -1338,7 +1307,7 @@ ngx_http_v2_state_headers(ngx_http_v2_connection_t *h2c, u_char *pos, h2c->closed_nodes--; } - stream = ngx_http_v2_create_stream(h2c, 0); + stream = ngx_http_v2_create_stream(h2c); if (stream == NULL) { return ngx_http_v2_connection_error(h2c, NGX_HTTP_V2_INTERNAL_ERROR); } @@ -2127,11 +2096,6 @@ ngx_http_v2_state_rst_stream(ngx_http_v2_connection_t *h2c, u_char *pos, "client canceled stream %ui", h2c->state.sid); break; - case NGX_HTTP_V2_REFUSED_STREAM: - ngx_log_error(NGX_LOG_INFO, fc->log, 0, - "client refused stream %ui", h2c->state.sid); - break; - case NGX_HTTP_V2_INTERNAL_ERROR: ngx_log_error(NGX_LOG_INFO, fc->log, 0, "client terminated stream %ui due to internal error", @@ -2199,7 +2163,6 @@ ngx_http_v2_state_settings_params(ngx_http_v2_connection_t *h2c, u_char *pos, { ssize_t window_delta; ngx_uint_t id, value; - ngx_http_v2_srv_conf_t *h2scf; ngx_http_v2_out_frame_t *frame; window_delta = 0; @@ -2261,14 +2224,6 @@ ngx_http_v2_state_settings_params(ngx_http_v2_connection_t *h2c, u_char *pos, NGX_HTTP_V2_PROTOCOL_ERROR); } - h2c->push_disabled = !value; - break; - - case NGX_HTTP_V2_MAX_STREAMS_SETTING: - h2scf = ngx_http_get_module_srv_conf(h2c->http_connection->conf_ctx, - ngx_http_v2_module); - - h2c->concurrent_pushes = ngx_min(value, h2scf->concurrent_pushes); break; case NGX_HTTP_V2_HEADER_TABLE_SIZE_SETTING: @@ -2723,163 +2678,6 @@ ngx_http_v2_parse_int(ngx_http_v2_connection_t *h2c, u_char **pos, u_char *end, } -ngx_http_v2_stream_t * -ngx_http_v2_push_stream(ngx_http_v2_stream_t *parent, ngx_str_t *path) -{ - ngx_int_t rc; - ngx_str_t value; - ngx_pool_t *pool; - ngx_uint_t index; - ngx_table_elt_t **h; - ngx_connection_t *fc; - ngx_http_request_t *r; - ngx_http_v2_node_t *node; - ngx_http_v2_stream_t *stream; - ngx_http_v2_srv_conf_t *h2scf; - ngx_http_v2_connection_t *h2c; - ngx_http_v2_parse_header_t *header; - - h2c = parent->connection; - - pool = ngx_create_pool(1024, h2c->connection->log); - if (pool == NULL) { - goto rst_stream; - } - - node = ngx_http_v2_get_node_by_id(h2c, h2c->last_push, 1); - - if (node == NULL) { - ngx_destroy_pool(pool); - goto rst_stream; - } - - stream = ngx_http_v2_create_stream(h2c, 1); - if (stream == NULL) { - - if (node->parent == NULL) { - h2scf = ngx_http_get_module_srv_conf(h2c->http_connection->conf_ctx, - ngx_http_v2_module); - - index = ngx_http_v2_index(h2scf, h2c->last_push); - h2c->streams_index[index] = node->index; - - ngx_queue_insert_tail(&h2c->closed, &node->reuse); - h2c->closed_nodes++; - } - - ngx_destroy_pool(pool); - goto rst_stream; - } - - if (node->parent) { - ngx_queue_remove(&node->reuse); - h2c->closed_nodes--; - } - - stream->pool = pool; - - r = stream->request; - fc = r->connection; - - stream->in_closed = 1; - stream->node = node; - - node->stream = stream; - - ngx_log_debug2(NGX_LOG_DEBUG_HTTP, h2c->connection->log, 0, - "http2 push stream sid:%ui " - "depends on %ui excl:0 weight:16", - h2c->last_push, parent->node->id); - - node->weight = NGX_HTTP_V2_DEFAULT_WEIGHT; - ngx_http_v2_set_dependency(h2c, node, parent->node->id, 0); - - r->method_name = ngx_http_core_get_method; - r->method = NGX_HTTP_GET; - - r->schema.data = ngx_pstrdup(pool, &parent->request->schema); - if (r->schema.data == NULL) { - goto close; - } - - r->schema.len = parent->request->schema.len; - - value.data = ngx_pstrdup(pool, path); - if (value.data == NULL) { - goto close; - } - - value.len = path->len; - - rc = ngx_http_v2_parse_path(r, &value); - - if (rc != NGX_OK) { - goto error; - } - - for (header = ngx_http_v2_parse_headers; header->name.len; header++) { - h = (ngx_table_elt_t **) - ((char *) &parent->request->headers_in + header->offset); - - if (*h == NULL) { - continue; - } - - value.len = (*h)->value.len; - - value.data = ngx_pnalloc(pool, value.len + 1); - if (value.data == NULL) { - goto close; - } - - ngx_memcpy(value.data, (*h)->value.data, value.len); - value.data[value.len] = '\0'; - - rc = ngx_http_v2_parse_header(r, header, &value); - - if (rc != NGX_OK) { - goto error; - } - } - - fc->write->handler = ngx_http_v2_run_request_handler; - ngx_post_event(fc->write, &ngx_posted_events); - - return stream; - -error: - - if (rc == NGX_ABORT) { - /* header handler has already finalized request */ - ngx_http_run_posted_requests(fc); - return NULL; - } - - if (rc == NGX_DECLINED) { - ngx_http_finalize_request(r, NGX_HTTP_BAD_REQUEST); - ngx_http_run_posted_requests(fc); - return NULL; - } - -close: - - ngx_http_v2_close_stream(stream, NGX_HTTP_INTERNAL_SERVER_ERROR); - - return NULL; - -rst_stream: - - if (ngx_http_v2_send_rst_stream(h2c, h2c->last_push, - NGX_HTTP_INTERNAL_SERVER_ERROR) - != NGX_OK) - { - h2c->connection->error = 1; - } - - return NULL; -} - - static ngx_int_t ngx_http_v2_send_settings(ngx_http_v2_connection_t *h2c) { @@ -3151,7 +2949,7 @@ ngx_http_v2_frame_handler(ngx_http_v2_connection_t *h2c, static ngx_http_v2_stream_t * -ngx_http_v2_create_stream(ngx_http_v2_connection_t *h2c, ngx_uint_t push) +ngx_http_v2_create_stream(ngx_http_v2_connection_t *h2c) { ngx_log_t *log; ngx_event_t *rev, *wev; @@ -3206,13 +3004,7 @@ ngx_http_v2_create_stream(ngx_http_v2_connection_t *h2c, ngx_uint_t push) ngx_memcpy(log, h2c->connection->log, sizeof(ngx_log_t)); log->data = ctx; - - if (push) { - log->action = "processing pushed request headers"; - - } else { - log->action = "reading client request headers"; - } + log->action = "reading client request headers"; ngx_memzero(rev, sizeof(ngx_event_t)); @@ -3284,12 +3076,7 @@ ngx_http_v2_create_stream(ngx_http_v2_connection_t *h2c, ngx_uint_t push) stream->send_window = h2c->init_window; stream->recv_window = h2scf->preread_size; - if (push) { - h2c->pushing++; - - } else { - h2c->processing++; - } + h2c->processing++; h2c->priority_limit += h2scf->concurrent_streams; @@ -3711,46 +3498,42 @@ ngx_http_v2_parse_scheme(ngx_http_request_t *r, ngx_str_t *value) static ngx_int_t ngx_http_v2_parse_authority(ngx_http_request_t *r, ngx_str_t *value) -{ - return ngx_http_v2_parse_header(r, &ngx_http_v2_parse_headers[0], value); -} - - -static ngx_int_t -ngx_http_v2_parse_header(ngx_http_request_t *r, - ngx_http_v2_parse_header_t *header, ngx_str_t *value) { ngx_table_elt_t *h; + ngx_http_header_t *hh; ngx_http_core_main_conf_t *cmcf; + static ngx_str_t host = ngx_string("host"); + h = ngx_list_push(&r->headers_in.headers); if (h == NULL) { return NGX_ERROR; } - h->key.len = header->name.len; - h->key.data = header->name.data; - h->lowcase_key = header->name.data; + h->hash = ngx_hash(ngx_hash(ngx_hash('h', 'o'), 's'), 't'); - if (header->hh == NULL) { - header->hash = ngx_hash_key(header->name.data, header->name.len); + h->key.len = host.len; + h->key.data = host.data; - cmcf = ngx_http_get_module_main_conf(r, ngx_http_core_module); + h->value.len = value->len; + h->value.data = value->data; - header->hh = ngx_hash_find(&cmcf->headers_in_hash, header->hash, - h->lowcase_key, h->key.len); - if (header->hh == NULL) { - return NGX_ERROR; - } - } + h->lowcase_key = host.data; - h->hash = header->hash; + cmcf = ngx_http_get_module_main_conf(r, ngx_http_core_module); - h->value.len = value->len; - h->value.data = value->data; + hh = ngx_hash_find(&cmcf->headers_in_hash, h->hash, + h->lowcase_key, h->key.len); + + if (hh == NULL) { + return NGX_ERROR; + } - if (header->hh->handler(r, h, header->hh->offset) != NGX_OK) { - /* header handler has already finalized request */ + if (hh->handler(r, h, hh->offset) != NGX_OK) { + /* + * request has been finalized already + * in ngx_http_process_host() + */ return NGX_ABORT; } @@ -3993,22 +3776,6 @@ ngx_http_v2_run_request(ngx_http_request_t *r) } -static void -ngx_http_v2_run_request_handler(ngx_event_t *ev) -{ - ngx_connection_t *fc; - ngx_http_request_t *r; - - fc = ev->data; - r = fc->data; - - ngx_log_debug0(NGX_LOG_DEBUG_HTTP, fc->log, 0, - "http2 run request handler"); - - ngx_http_v2_run_request(r); -} - - ngx_int_t ngx_http_v2_read_request_body(ngx_http_request_t *r) { @@ -4612,7 +4379,6 @@ void ngx_http_v2_close_stream(ngx_http_v2_stream_t *stream, ngx_int_t rc) { ngx_pool_t *pool; - ngx_uint_t push; ngx_event_t *ev; ngx_connection_t *fc; ngx_http_v2_node_t *node; @@ -4621,10 +4387,9 @@ ngx_http_v2_close_stream(ngx_http_v2_stream_t *stream, ngx_int_t rc) h2c = stream->connection; node = stream->node; - ngx_log_debug4(NGX_LOG_DEBUG_HTTP, h2c->connection->log, 0, - "http2 close stream %ui, queued %ui, " - "processing %ui, pushing %ui", - node->id, stream->queued, h2c->processing, h2c->pushing); + ngx_log_debug3(NGX_LOG_DEBUG_HTTP, h2c->connection->log, 0, + "http2 close stream %ui, queued %ui, processing %ui", + node->id, stream->queued, h2c->processing); fc = stream->request->connection; @@ -4659,8 +4424,6 @@ ngx_http_v2_close_stream(ngx_http_v2_stream_t *stream, ngx_int_t rc) h2c->state.stream = NULL; } - push = stream->node->id % 2 == 0; - node->stream = NULL; ngx_queue_insert_tail(&h2c->closed, &node->reuse); @@ -4710,14 +4473,9 @@ ngx_http_v2_close_stream(ngx_http_v2_stream_t *stream, ngx_int_t rc) fc->data = h2c->free_fake_connections; h2c->free_fake_connections = fc; - if (push) { - h2c->pushing--; - - } else { - h2c->processing--; - } + h2c->processing--; - if (h2c->processing || h2c->pushing || h2c->blocked) { + if (h2c->processing || h2c->blocked) { return; } @@ -4893,7 +4651,7 @@ ngx_http_v2_finalize_connection(ngx_http_v2_connection_t *h2c, } } - if (!h2c->processing && !h2c->pushing) { + if (!h2c->processing) { goto done; } @@ -4941,7 +4699,7 @@ ngx_http_v2_finalize_connection(ngx_http_v2_connection_t *h2c, h2c->blocked = 0; - if (h2c->processing || h2c->pushing) { + if (h2c->processing) { c->error = 1; return; } diff --git a/src/http/v2/ngx_http_v2.h b/src/http/v2/ngx_http_v2.h index cb7a313e01b..cb9014ccfb1 100644 --- a/src/http/v2/ngx_http_v2.h +++ b/src/http/v2/ngx_http_v2.h @@ -24,8 +24,6 @@ #define NGX_HTTP_V2_MAX_FIELD \ (127 + (1 << (NGX_HTTP_V2_INT_OCTETS - 1) * 7) - 1) -#define NGX_HTTP_V2_STREAM_ID_SIZE 4 - #define NGX_HTTP_V2_FRAME_HEADER_SIZE 9 /* frame types */ @@ -67,7 +65,6 @@ typedef struct { ngx_flag_t enable; size_t pool_size; ngx_uint_t concurrent_streams; - ngx_uint_t concurrent_pushes; size_t preread_size; ngx_uint_t streams_index_mask; } ngx_http_v2_srv_conf_t; @@ -136,9 +133,6 @@ struct ngx_http_v2_connection_s { ngx_uint_t idle; ngx_uint_t priority_limit; - ngx_uint_t pushing; - ngx_uint_t concurrent_pushes; - size_t send_window; size_t recv_window; size_t init_window; @@ -165,7 +159,6 @@ struct ngx_http_v2_connection_s { ngx_uint_t closed_nodes; ngx_uint_t last_sid; - ngx_uint_t last_push; time_t lingering_time; @@ -173,7 +166,6 @@ struct ngx_http_v2_connection_s { unsigned table_update:1; unsigned blocked:1; unsigned goaway:1; - unsigned push_disabled:1; }; @@ -303,9 +295,6 @@ void ngx_http_v2_init(ngx_event_t *rev); ngx_int_t ngx_http_v2_read_request_body(ngx_http_request_t *r); ngx_int_t ngx_http_v2_read_unbuffered_request_body(ngx_http_request_t *r); -ngx_http_v2_stream_t *ngx_http_v2_push_stream(ngx_http_v2_stream_t *parent, - ngx_str_t *path); - void ngx_http_v2_close_stream(ngx_http_v2_stream_t *stream, ngx_int_t rc); ngx_int_t ngx_http_v2_send_output_queue(ngx_http_v2_connection_t *h2c); @@ -407,15 +396,12 @@ ngx_int_t ngx_http_v2_table_size(ngx_http_v2_connection_t *h2c, size_t size); #define NGX_HTTP_V2_STATUS_404_INDEX 13 #define NGX_HTTP_V2_STATUS_500_INDEX 14 -#define NGX_HTTP_V2_ACCEPT_ENCODING_INDEX 16 -#define NGX_HTTP_V2_ACCEPT_LANGUAGE_INDEX 17 #define NGX_HTTP_V2_CONTENT_LENGTH_INDEX 28 #define NGX_HTTP_V2_CONTENT_TYPE_INDEX 31 #define NGX_HTTP_V2_DATE_INDEX 33 #define NGX_HTTP_V2_LAST_MODIFIED_INDEX 44 #define NGX_HTTP_V2_LOCATION_INDEX 46 #define NGX_HTTP_V2_SERVER_INDEX 54 -#define NGX_HTTP_V2_USER_AGENT_INDEX 58 #define NGX_HTTP_V2_VARY_INDEX 59 #define NGX_HTTP_V2_PREFACE_START "PRI * HTTP/2.0\r\n" diff --git a/src/http/v2/ngx_http_v2_filter_module.c b/src/http/v2/ngx_http_v2_filter_module.c index 5f8626a40be..1e2cafaf145 100644 --- a/src/http/v2/ngx_http_v2_filter_module.c +++ b/src/http/v2/ngx_http_v2_filter_module.c @@ -27,39 +27,8 @@ #define NGX_HTTP_V2_NO_TRAILERS (ngx_http_v2_out_frame_t *) -1 -typedef struct { - ngx_str_t name; - u_char index; - ngx_uint_t offset; -} ngx_http_v2_push_header_t; - - -static ngx_http_v2_push_header_t ngx_http_v2_push_headers[] = { - { ngx_string(":authority"), NGX_HTTP_V2_AUTHORITY_INDEX, - offsetof(ngx_http_headers_in_t, host) }, - - { ngx_string("accept-encoding"), NGX_HTTP_V2_ACCEPT_ENCODING_INDEX, - offsetof(ngx_http_headers_in_t, accept_encoding) }, - - { ngx_string("accept-language"), NGX_HTTP_V2_ACCEPT_LANGUAGE_INDEX, - offsetof(ngx_http_headers_in_t, accept_language) }, - - { ngx_string("user-agent"), NGX_HTTP_V2_USER_AGENT_INDEX, - offsetof(ngx_http_headers_in_t, user_agent) }, -}; - -#define NGX_HTTP_V2_PUSH_HEADERS \ - (sizeof(ngx_http_v2_push_headers) / sizeof(ngx_http_v2_push_header_t)) - - -static ngx_int_t ngx_http_v2_push_resources(ngx_http_request_t *r); -static ngx_int_t ngx_http_v2_push_resource(ngx_http_request_t *r, - ngx_str_t *path, ngx_str_t *binary); - static ngx_http_v2_out_frame_t *ngx_http_v2_create_headers_frame( ngx_http_request_t *r, u_char *pos, u_char *end, ngx_uint_t fin); -static ngx_http_v2_out_frame_t *ngx_http_v2_create_push_frame( - ngx_http_request_t *r, u_char *pos, u_char *end); static ngx_http_v2_out_frame_t *ngx_http_v2_create_trailers_frame( ngx_http_request_t *r); @@ -82,8 +51,6 @@ static ngx_inline ngx_int_t ngx_http_v2_filter_send( static ngx_int_t ngx_http_v2_headers_frame_handler( ngx_http_v2_connection_t *h2c, ngx_http_v2_out_frame_t *frame); -static ngx_int_t ngx_http_v2_push_frame_handler( - ngx_http_v2_connection_t *h2c, ngx_http_v2_out_frame_t *frame); static ngx_int_t ngx_http_v2_data_frame_handler( ngx_http_v2_connection_t *h2c, ngx_http_v2_out_frame_t *frame); static ngx_inline void ngx_http_v2_handle_frame( @@ -244,15 +211,6 @@ ngx_http_v2_header_filter(ngx_http_request_t *r) h2c = stream->connection; - if (!h2c->push_disabled && !h2c->goaway - && stream->node->id % 2 == 1 - && r->method != NGX_HTTP_HEAD) - { - if (ngx_http_v2_push_resources(r) != NGX_OK) { - return NGX_ERROR; - } - } - len = h2c->table_update ? 1 : 0; len += status ? 1 : 1 + ngx_http_v2_literal_size("418"); @@ -653,7 +611,7 @@ ngx_http_v2_header_filter(ngx_http_request_t *r) ngx_http_v2_queue_blocked_frame(h2c, frame); - stream->queued++; + stream->queued = 1; cln = ngx_http_cleanup_add(r, 0); if (cln == NULL) { @@ -671,409 +629,6 @@ ngx_http_v2_header_filter(ngx_http_request_t *r) } -static ngx_int_t -ngx_http_v2_push_resources(ngx_http_request_t *r) -{ - u_char *start, *end, *last; - ngx_int_t rc; - ngx_str_t path; - ngx_uint_t i, push; - ngx_table_elt_t *h; - ngx_http_v2_loc_conf_t *h2lcf; - ngx_http_complex_value_t *pushes; - ngx_str_t binary[NGX_HTTP_V2_PUSH_HEADERS]; - - ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, - "http2 push resources"); - - ngx_memzero(binary, NGX_HTTP_V2_PUSH_HEADERS * sizeof(ngx_str_t)); - - h2lcf = ngx_http_get_module_loc_conf(r, ngx_http_v2_module); - - if (h2lcf->pushes) { - pushes = h2lcf->pushes->elts; - - for (i = 0; i < h2lcf->pushes->nelts; i++) { - - if (ngx_http_complex_value(r, &pushes[i], &path) != NGX_OK) { - return NGX_ERROR; - } - - if (path.len == 0) { - continue; - } - - if (path.len == 3 && ngx_strncmp(path.data, "off", 3) == 0) { - continue; - } - - rc = ngx_http_v2_push_resource(r, &path, binary); - - if (rc == NGX_ERROR) { - return NGX_ERROR; - } - - if (rc == NGX_ABORT) { - return NGX_OK; - } - - /* NGX_OK, NGX_DECLINED */ - } - } - - if (!h2lcf->push_preload) { - return NGX_OK; - } - - for (h = r->headers_out.link; h; h = h->next) { - - ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, - "http2 parse link: \"%V\"", &h->value); - - start = h->value.data; - end = h->value.data + h->value.len; - - next_link: - - while (start < end && *start == ' ') { start++; } - - if (start == end || *start++ != '<') { - continue; - } - - while (start < end && *start == ' ') { start++; } - - for (last = start; last < end && *last != '>'; last++) { - /* void */ - } - - if (last == start || last == end) { - continue; - } - - path.len = last - start; - path.data = start; - - start = last + 1; - - while (start < end && *start == ' ') { start++; } - - if (start == end) { - continue; - } - - if (*start == ',') { - start++; - goto next_link; - } - - if (*start++ != ';') { - continue; - } - - last = ngx_strlchr(start, end, ','); - - if (last == NULL) { - last = end; - } - - push = 0; - - for ( ;; ) { - - while (start < last && *start == ' ') { start++; } - - if (last - start >= 6 - && ngx_strncasecmp(start, (u_char *) "nopush", 6) == 0) - { - start += 6; - - if (start == last || *start == ' ' || *start == ';') { - push = 0; - break; - } - - goto next_param; - } - - if (last - start >= 11 - && ngx_strncasecmp(start, (u_char *) "rel=preload", 11) == 0) - { - start += 11; - - if (start == last || *start == ' ' || *start == ';') { - push = 1; - } - - goto next_param; - } - - if (last - start >= 4 - && ngx_strncasecmp(start, (u_char *) "rel=", 4) == 0) - { - start += 4; - - while (start < last && *start == ' ') { start++; } - - if (start == last || *start++ != '"') { - goto next_param; - } - - for ( ;; ) { - - while (start < last && *start == ' ') { start++; } - - if (last - start >= 7 - && ngx_strncasecmp(start, (u_char *) "preload", 7) == 0) - { - start += 7; - - if (start < last && (*start == ' ' || *start == '"')) { - push = 1; - break; - } - } - - while (start < last && *start != ' ' && *start != '"') { - start++; - } - - if (start == last) { - break; - } - - if (*start == '"') { - break; - } - - start++; - } - } - - next_param: - - start = ngx_strlchr(start, last, ';'); - - if (start == NULL) { - break; - } - - start++; - } - - if (push) { - while (path.len && path.data[path.len - 1] == ' ') { - path.len--; - } - } - - if (push && path.len - && !(path.len > 1 && path.data[0] == '/' && path.data[1] == '/')) - { - rc = ngx_http_v2_push_resource(r, &path, binary); - - if (rc == NGX_ERROR) { - return NGX_ERROR; - } - - if (rc == NGX_ABORT) { - return NGX_OK; - } - - /* NGX_OK, NGX_DECLINED */ - } - - if (last < end) { - start = last + 1; - goto next_link; - } - } - - return NGX_OK; -} - - -static ngx_int_t -ngx_http_v2_push_resource(ngx_http_request_t *r, ngx_str_t *path, - ngx_str_t *binary) -{ - u_char *start, *pos, *tmp; - size_t len; - ngx_str_t *value; - ngx_uint_t i; - ngx_table_elt_t **h; - ngx_connection_t *fc; - ngx_http_v2_stream_t *stream; - ngx_http_v2_out_frame_t *frame; - ngx_http_v2_connection_t *h2c; - ngx_http_v2_push_header_t *ph; - - fc = r->connection; - - ngx_log_debug0(NGX_LOG_DEBUG_HTTP, fc->log, 0, "http2 push resource"); - - stream = r->stream; - h2c = stream->connection; - - if (!ngx_path_separator(path->data[0])) { - ngx_log_error(NGX_LOG_WARN, fc->log, 0, - "non-absolute path \"%V\" not pushed", path); - return NGX_DECLINED; - } - - ngx_log_debug2(NGX_LOG_DEBUG_HTTP, h2c->connection->log, 0, - "http2 pushing:%ui limit:%ui", - h2c->pushing, h2c->concurrent_pushes); - - if (h2c->pushing >= h2c->concurrent_pushes) { - return NGX_ABORT; - } - - if (h2c->last_push == 0x7ffffffe) { - return NGX_ABORT; - } - - if (path->len > NGX_HTTP_V2_MAX_FIELD) { - return NGX_DECLINED; - } - - if (r->headers_in.host == NULL) { - return NGX_ABORT; - } - - ph = ngx_http_v2_push_headers; - - len = ngx_max(r->schema.len, path->len); - - if (binary[0].len) { - tmp = ngx_palloc(r->pool, len); - if (tmp == NULL) { - return NGX_ERROR; - } - - } else { - for (i = 0; i < NGX_HTTP_V2_PUSH_HEADERS; i++) { - h = (ngx_table_elt_t **) ((char *) &r->headers_in + ph[i].offset); - - if (*h) { - len = ngx_max(len, (*h)->value.len); - } - } - - tmp = ngx_palloc(r->pool, len); - if (tmp == NULL) { - return NGX_ERROR; - } - - for (i = 0; i < NGX_HTTP_V2_PUSH_HEADERS; i++) { - h = (ngx_table_elt_t **) ((char *) &r->headers_in + ph[i].offset); - - if (*h == NULL) { - continue; - } - - value = &(*h)->value; - - len = 1 + NGX_HTTP_V2_INT_OCTETS + value->len; - - pos = ngx_pnalloc(r->pool, len); - if (pos == NULL) { - return NGX_ERROR; - } - - binary[i].data = pos; - - *pos++ = ngx_http_v2_inc_indexed(ph[i].index); - pos = ngx_http_v2_write_value(pos, value->data, value->len, tmp); - - binary[i].len = pos - binary[i].data; - } - } - - len = (h2c->table_update ? 1 : 0) - + 1 - + 1 + NGX_HTTP_V2_INT_OCTETS + path->len - + 1 + NGX_HTTP_V2_INT_OCTETS + r->schema.len; - - for (i = 0; i < NGX_HTTP_V2_PUSH_HEADERS; i++) { - len += binary[i].len; - } - - pos = ngx_pnalloc(r->pool, len); - if (pos == NULL) { - return NGX_ERROR; - } - - start = pos; - - if (h2c->table_update) { - ngx_log_debug0(NGX_LOG_DEBUG_HTTP, fc->log, 0, - "http2 table size update: 0"); - *pos++ = (1 << 5) | 0; - h2c->table_update = 0; - } - - ngx_log_debug0(NGX_LOG_DEBUG_HTTP, fc->log, 0, - "http2 push header: \":method: GET\""); - - *pos++ = ngx_http_v2_indexed(NGX_HTTP_V2_METHOD_GET_INDEX); - - ngx_log_debug1(NGX_LOG_DEBUG_HTTP, fc->log, 0, - "http2 push header: \":path: %V\"", path); - - *pos++ = ngx_http_v2_inc_indexed(NGX_HTTP_V2_PATH_INDEX); - pos = ngx_http_v2_write_value(pos, path->data, path->len, tmp); - - ngx_log_debug1(NGX_LOG_DEBUG_HTTP, fc->log, 0, - "http2 push header: \":scheme: %V\"", &r->schema); - - if (r->schema.len == 5 && ngx_strncmp(r->schema.data, "https", 5) == 0) { - *pos++ = ngx_http_v2_indexed(NGX_HTTP_V2_SCHEME_HTTPS_INDEX); - - } else if (r->schema.len == 4 - && ngx_strncmp(r->schema.data, "http", 4) == 0) - { - *pos++ = ngx_http_v2_indexed(NGX_HTTP_V2_SCHEME_HTTP_INDEX); - - } else { - *pos++ = ngx_http_v2_inc_indexed(NGX_HTTP_V2_SCHEME_HTTP_INDEX); - pos = ngx_http_v2_write_value(pos, r->schema.data, r->schema.len, tmp); - } - - for (i = 0; i < NGX_HTTP_V2_PUSH_HEADERS; i++) { - h = (ngx_table_elt_t **) ((char *) &r->headers_in + ph[i].offset); - - if (*h == NULL) { - continue; - } - - ngx_log_debug2(NGX_LOG_DEBUG_HTTP, fc->log, 0, - "http2 push header: \"%V: %V\"", - &ph[i].name, &(*h)->value); - - pos = ngx_cpymem(pos, binary[i].data, binary[i].len); - } - - frame = ngx_http_v2_create_push_frame(r, start, pos); - if (frame == NULL) { - return NGX_ERROR; - } - - ngx_http_v2_queue_blocked_frame(h2c, frame); - - stream->queued++; - - stream = ngx_http_v2_push_stream(stream, path); - - if (stream) { - stream->request->request_length = pos - start; - return NGX_OK; - } - - return NGX_ERROR; -} - - static ngx_http_v2_out_frame_t * ngx_http_v2_create_headers_frame(ngx_http_request_t *r, u_char *pos, u_char *end, ngx_uint_t fin) @@ -1179,125 +734,6 @@ ngx_http_v2_create_headers_frame(ngx_http_request_t *r, u_char *pos, } -static ngx_http_v2_out_frame_t * -ngx_http_v2_create_push_frame(ngx_http_request_t *r, u_char *pos, u_char *end) -{ - u_char type, flags; - size_t rest, frame_size, len; - ngx_buf_t *b; - ngx_chain_t *cl, **ll; - ngx_http_v2_stream_t *stream; - ngx_http_v2_out_frame_t *frame; - ngx_http_v2_connection_t *h2c; - - stream = r->stream; - h2c = stream->connection; - rest = NGX_HTTP_V2_STREAM_ID_SIZE + (end - pos); - - frame = ngx_palloc(r->pool, sizeof(ngx_http_v2_out_frame_t)); - if (frame == NULL) { - return NULL; - } - - frame->handler = ngx_http_v2_push_frame_handler; - frame->stream = stream; - frame->length = rest; - frame->blocked = 1; - frame->fin = 0; - - ll = &frame->first; - - type = NGX_HTTP_V2_PUSH_PROMISE_FRAME; - flags = NGX_HTTP_V2_NO_FLAG; - frame_size = h2c->frame_size; - - for ( ;; ) { - if (rest <= frame_size) { - frame_size = rest; - flags |= NGX_HTTP_V2_END_HEADERS_FLAG; - } - - b = ngx_create_temp_buf(r->pool, - NGX_HTTP_V2_FRAME_HEADER_SIZE - + ((type == NGX_HTTP_V2_PUSH_PROMISE_FRAME) - ? NGX_HTTP_V2_STREAM_ID_SIZE : 0)); - if (b == NULL) { - return NULL; - } - - b->last = ngx_http_v2_write_len_and_type(b->last, frame_size, type); - *b->last++ = flags; - b->last = ngx_http_v2_write_sid(b->last, stream->node->id); - - b->tag = (ngx_buf_tag_t) &ngx_http_v2_module; - - if (type == NGX_HTTP_V2_PUSH_PROMISE_FRAME) { - h2c->last_push += 2; - - b->last = ngx_http_v2_write_sid(b->last, h2c->last_push); - len = frame_size - NGX_HTTP_V2_STREAM_ID_SIZE; - - } else { - len = frame_size; - } - - cl = ngx_alloc_chain_link(r->pool); - if (cl == NULL) { - return NULL; - } - - cl->buf = b; - - *ll = cl; - ll = &cl->next; - - b = ngx_calloc_buf(r->pool); - if (b == NULL) { - return NULL; - } - - b->pos = pos; - - pos += len; - - b->last = pos; - b->start = b->pos; - b->end = b->last; - b->temporary = 1; - - cl = ngx_alloc_chain_link(r->pool); - if (cl == NULL) { - return NULL; - } - - cl->buf = b; - - *ll = cl; - ll = &cl->next; - - rest -= frame_size; - - if (rest) { - frame->length += NGX_HTTP_V2_FRAME_HEADER_SIZE; - - type = NGX_HTTP_V2_CONTINUATION_FRAME; - continue; - } - - cl->next = NULL; - frame->last = cl; - - ngx_log_debug4(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, - "http2:%ui create PUSH_PROMISE frame %p: " - "sid:%ui len:%uz", - stream->node->id, frame, h2c->last_push, - frame->length); - - return frame; - } -} - - static ngx_http_v2_out_frame_t * ngx_http_v2_create_trailers_frame(ngx_http_request_t *r) { @@ -1901,62 +1337,6 @@ ngx_http_v2_headers_frame_handler(ngx_http_v2_connection_t *h2c, } -static ngx_int_t -ngx_http_v2_push_frame_handler(ngx_http_v2_connection_t *h2c, - ngx_http_v2_out_frame_t *frame) -{ - ngx_chain_t *cl, *ln; - ngx_http_v2_stream_t *stream; - - stream = frame->stream; - cl = frame->first; - - for ( ;; ) { - if (cl->buf->pos != cl->buf->last) { - frame->first = cl; - - ngx_log_debug2(NGX_LOG_DEBUG_HTTP, h2c->connection->log, 0, - "http2:%ui PUSH_PROMISE frame %p was sent partially", - stream->node->id, frame); - - return NGX_AGAIN; - } - - ln = cl->next; - - if (cl->buf->tag == (ngx_buf_tag_t) &ngx_http_v2_module) { - cl->next = stream->free_frame_headers; - stream->free_frame_headers = cl; - - } else { - cl->next = stream->free_bufs; - stream->free_bufs = cl; - } - - if (cl == frame->last) { - break; - } - - cl = ln; - } - - ngx_log_debug2(NGX_LOG_DEBUG_HTTP, h2c->connection->log, 0, - "http2:%ui PUSH_PROMISE frame %p was sent", - stream->node->id, frame); - - stream->request->header_size += NGX_HTTP_V2_FRAME_HEADER_SIZE - + frame->length; - - h2c->payload_bytes += frame->length; - - ngx_http_v2_handle_frame(stream, frame); - - ngx_http_v2_handle_stream(h2c, stream); - - return NGX_OK; -} - - static ngx_int_t ngx_http_v2_data_frame_handler(ngx_http_v2_connection_t *h2c, ngx_http_v2_out_frame_t *frame) diff --git a/src/http/v2/ngx_http_v2_module.c b/src/http/v2/ngx_http_v2_module.c index 09396a50b6f..62af9a54357 100644 --- a/src/http/v2/ngx_http_v2_module.c +++ b/src/http/v2/ngx_http_v2_module.c @@ -27,8 +27,6 @@ static void *ngx_http_v2_create_loc_conf(ngx_conf_t *cf); static char *ngx_http_v2_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child); -static char *ngx_http_v2_push(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); - static char *ngx_http_v2_recv_buffer_size(ngx_conf_t *cf, void *post, void *data); static char *ngx_http_v2_pool_size(ngx_conf_t *cf, void *post, void *data); @@ -105,9 +103,9 @@ static ngx_command_t ngx_http_v2_commands[] = { { ngx_string("http2_max_concurrent_pushes"), NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1, - ngx_conf_set_num_slot, - NGX_HTTP_SRV_CONF_OFFSET, - offsetof(ngx_http_v2_srv_conf_t, concurrent_pushes), + ngx_http_v2_obsolete, + 0, + 0, NULL }, { ngx_string("http2_max_requests"), @@ -168,15 +166,15 @@ static ngx_command_t ngx_http_v2_commands[] = { { ngx_string("http2_push_preload"), NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG, - ngx_conf_set_flag_slot, - NGX_HTTP_LOC_CONF_OFFSET, - offsetof(ngx_http_v2_loc_conf_t, push_preload), + ngx_http_v2_obsolete, + 0, + 0, NULL }, { ngx_string("http2_push"), NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, - ngx_http_v2_push, - NGX_HTTP_LOC_CONF_OFFSET, + ngx_http_v2_obsolete, + 0, 0, NULL }, @@ -326,7 +324,6 @@ ngx_http_v2_create_srv_conf(ngx_conf_t *cf) h2scf->pool_size = NGX_CONF_UNSET_SIZE; h2scf->concurrent_streams = NGX_CONF_UNSET_UINT; - h2scf->concurrent_pushes = NGX_CONF_UNSET_UINT; h2scf->preread_size = NGX_CONF_UNSET_SIZE; @@ -348,8 +345,6 @@ ngx_http_v2_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) ngx_conf_merge_uint_value(conf->concurrent_streams, prev->concurrent_streams, 128); - ngx_conf_merge_uint_value(conf->concurrent_pushes, - prev->concurrent_pushes, 10); ngx_conf_merge_size_value(conf->preread_size, prev->preread_size, 65536); @@ -370,17 +365,8 @@ ngx_http_v2_create_loc_conf(ngx_conf_t *cf) return NULL; } - /* - * set by ngx_pcalloc(): - * - * h2lcf->pushes = NULL; - */ - h2lcf->chunk_size = NGX_CONF_UNSET_SIZE; - h2lcf->push_preload = NGX_CONF_UNSET; - h2lcf->push = NGX_CONF_UNSET; - return h2lcf; } @@ -393,72 +379,6 @@ ngx_http_v2_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) ngx_conf_merge_size_value(conf->chunk_size, prev->chunk_size, 8 * 1024); - ngx_conf_merge_value(conf->push, prev->push, 1); - - if (conf->push && conf->pushes == NULL) { - conf->pushes = prev->pushes; - } - - ngx_conf_merge_value(conf->push_preload, prev->push_preload, 0); - - return NGX_CONF_OK; -} - - -static char * -ngx_http_v2_push(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) -{ - ngx_http_v2_loc_conf_t *h2lcf = conf; - - ngx_str_t *value; - ngx_http_complex_value_t *cv; - ngx_http_compile_complex_value_t ccv; - - value = cf->args->elts; - - if (ngx_strcmp(value[1].data, "off") == 0) { - - if (h2lcf->pushes) { - return "\"off\" parameter cannot be used with URI"; - } - - if (h2lcf->push == 0) { - return "is duplicate"; - } - - h2lcf->push = 0; - return NGX_CONF_OK; - } - - if (h2lcf->push == 0) { - return "URI cannot be used with \"off\" parameter"; - } - - h2lcf->push = 1; - - if (h2lcf->pushes == NULL) { - h2lcf->pushes = ngx_array_create(cf->pool, 1, - sizeof(ngx_http_complex_value_t)); - if (h2lcf->pushes == NULL) { - return NGX_CONF_ERROR; - } - } - - cv = ngx_array_push(h2lcf->pushes); - if (cv == NULL) { - return NGX_CONF_ERROR; - } - - ngx_memzero(&ccv, sizeof(ngx_http_compile_complex_value_t)); - - ccv.cf = cf; - ccv.value = &value[1]; - ccv.complex_value = cv; - - if (ngx_http_compile_complex_value(&ccv) != NGX_OK) { - return NGX_CONF_ERROR; - } - return NGX_CONF_OK; } @@ -562,10 +482,17 @@ ngx_http_v2_obsolete(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) { ngx_conf_deprecated_t *d = cmd->post; - ngx_conf_log_error(NGX_LOG_WARN, cf, 0, - "the \"%s\" directive is obsolete, " - "use the \"%s\" directive instead", - d->old_name, d->new_name); + if (d) { + ngx_conf_log_error(NGX_LOG_WARN, cf, 0, + "the \"%s\" directive is obsolete, " + "use the \"%s\" directive instead", + d->old_name, d->new_name); + + } else { + ngx_conf_log_error(NGX_LOG_WARN, cf, 0, + "the \"%V\" directive is obsolete, ignored", + &cmd->name); + } return NGX_CONF_OK; } diff --git a/src/http/v2/ngx_http_v2_module.h b/src/http/v2/ngx_http_v2_module.h index 22a2d84ca09..07a595cfdd3 100644 --- a/src/http/v2/ngx_http_v2_module.h +++ b/src/http/v2/ngx_http_v2_module.h @@ -22,11 +22,6 @@ typedef struct { typedef struct { size_t chunk_size; - - ngx_flag_t push_preload; - - ngx_flag_t push; - ngx_array_t *pushes; } ngx_http_v2_loc_conf_t; From e3d2bd0a10390c5f18f9d723164dcfa16395b9bf Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Mon, 12 Jun 2023 23:38:56 +0400 Subject: [PATCH 006/279] QUIC: fixed rttvar on subsequent RTT samples (ticket #2505). Previously, computing rttvar used an updated smoothed_rtt value as per RFC 9002, section 5.3, which appears to be specified in a wrong order. A technical errata ID 7539 is reported. --- src/event/quic/ngx_event_quic_ack.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/event/quic/ngx_event_quic_ack.c b/src/event/quic/ngx_event_quic_ack.c index 21ee888c9cb..f194abfffdb 100644 --- a/src/event/quic/ngx_event_quic_ack.c +++ b/src/event/quic/ngx_event_quic_ack.c @@ -207,9 +207,9 @@ ngx_quic_rtt_sample(ngx_connection_t *c, ngx_quic_ack_frame_t *ack, adjusted_rtt -= ack_delay; } - qc->avg_rtt += (adjusted_rtt >> 3) - (qc->avg_rtt >> 3); rttvar_sample = ngx_abs((ngx_msec_int_t) (qc->avg_rtt - adjusted_rtt)); qc->rttvar += (rttvar_sample >> 2) - (qc->rttvar >> 2); + qc->avg_rtt += (adjusted_rtt >> 3) - (qc->avg_rtt >> 3); } ngx_log_debug4(NGX_LOG_DEBUG_EVENT, c->log, 0, From 73a872977dd4094d9ea3f3ba086707d526b7e61b Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Tue, 13 Jun 2023 18:08:09 +0300 Subject: [PATCH 007/279] nginx-1.25.1-RELEASE --- docs/xml/nginx/changes.xml | 43 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/docs/xml/nginx/changes.xml b/docs/xml/nginx/changes.xml index b30f6430844..1d199361fed 100644 --- a/docs/xml/nginx/changes.xml +++ b/docs/xml/nginx/changes.xml @@ -5,6 +5,49 @@ + + + + +директива http2, позволяющая включать HTTP/2 в отдельных блоках server; +параметр http2 директивы listen объявлен устаревшим. + + +the "http2" directive, which enables HTTP/2 on a per-server basis; +the "http2" parameter of the "listen" directive is now deprecated. + + + + + +поддержка HTTP/2 server push упразднена. + + +HTTP/2 server push support has been removed. + + + + + +устаревшая директива ssl больше не поддерживается. + + +the deprecated "ssl" directive is not supported anymore. + + + + + +в HTTP/3 при использовании OpenSSL. + + +in HTTP/3 when using OpenSSL. + + + + + + From 6af8fe9cc44dbef43af4db5ea09d54ebd4aadbc5 Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Tue, 13 Jun 2023 18:08:10 +0300 Subject: [PATCH 008/279] release-1.25.1 tag --- .hgtags | 1 + 1 file changed, 1 insertion(+) diff --git a/.hgtags b/.hgtags index 4655010fe71..cfa3b321e51 100644 --- a/.hgtags +++ b/.hgtags @@ -473,3 +473,4 @@ aa901551a7ebad1e8b0f8c11cb44e3424ba29707 release-1.23.2 ff3afd1ce6a6b65057741df442adfaa71a0e2588 release-1.23.3 ac779115ed6ee4f3039e9aea414a54e560450ee2 release-1.23.4 12dcf92b0c2c68552398f19644ce3104459807d7 release-1.25.0 +f8134640e8615448205785cf00b0bc810489b495 release-1.25.1 From 8b28fd7f26fe046007e8eb90ad7120ae033b7c58 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Tue, 20 Jun 2023 17:01:00 +0400 Subject: [PATCH 009/279] Version bump. --- src/core/nginx.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/nginx.h b/src/core/nginx.h index 7a32ef19124..dba95c4a19a 100644 --- a/src/core/nginx.h +++ b/src/core/nginx.h @@ -9,8 +9,8 @@ #define _NGINX_H_INCLUDED_ -#define nginx_version 1025001 -#define NGINX_VERSION "1.25.1" +#define nginx_version 1025002 +#define NGINX_VERSION "1.25.2" #define NGINX_VER "nginx/" NGINX_VERSION #ifdef NGX_BUILD From 58c11ee714856cfe62bd52d9b2d0a52401c1abfc Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Fri, 9 Jun 2023 10:25:54 +0400 Subject: [PATCH 010/279] QUIC: a new constant for AEAD tag length. Previously used constant EVP_GCM_TLS_TAG_LEN had misleading name since it was used not only with GCM, but also with CHACHAPOLY. Now a new constant NGX_QUIC_TAG_LEN introduced. Luckily all AEAD algorithms used by QUIC have the same tag length of 16. --- src/event/quic/ngx_event_quic_openssl_compat.c | 6 +++--- src/event/quic/ngx_event_quic_protection.c | 18 +++++++++--------- src/event/quic/ngx_event_quic_protection.h | 3 ++- src/event/quic/ngx_event_quic_transport.c | 6 +++--- 4 files changed, 17 insertions(+), 16 deletions(-) diff --git a/src/event/quic/ngx_event_quic_openssl_compat.c b/src/event/quic/ngx_event_quic_openssl_compat.c index 63d380e35bd..318feda10f3 100644 --- a/src/event/quic/ngx_event_quic_openssl_compat.c +++ b/src/event/quic/ngx_event_quic_openssl_compat.c @@ -445,7 +445,7 @@ SSL_provide_quic_data(SSL *ssl, enum ssl_encryption_level_t level, u_char in[NGX_QUIC_COMPAT_RECORD_SIZE + 1]; u_char out[NGX_QUIC_COMPAT_RECORD_SIZE + 1 + SSL3_RT_HEADER_LENGTH - + EVP_GCM_TLS_TAG_LEN]; + + NGX_QUIC_TAG_LEN]; c = ngx_ssl_get_connection(ssl); @@ -528,7 +528,7 @@ ngx_quic_compat_create_header(ngx_quic_compat_record_t *rec, u_char *out, } else { type = SSL3_RT_APPLICATION_DATA; - len += EVP_GCM_TLS_TAG_LEN; + len += NGX_QUIC_TAG_LEN; } out[0] = type; @@ -552,7 +552,7 @@ ngx_quic_compat_create_record(ngx_quic_compat_record_t *rec, ngx_str_t *res) ad.data = res->data; ad.len = ngx_quic_compat_create_header(rec, ad.data, 0); - out.len = rec->payload.len + EVP_GCM_TLS_TAG_LEN; + out.len = rec->payload.len + NGX_QUIC_TAG_LEN; out.data = res->data + ad.len; #ifdef NGX_QUIC_DEBUG_CRYPTO diff --git a/src/event/quic/ngx_event_quic_protection.c b/src/event/quic/ngx_event_quic_protection.c index 7f772016b04..9b967ab0f28 100644 --- a/src/event/quic/ngx_event_quic_protection.c +++ b/src/event/quic/ngx_event_quic_protection.c @@ -406,7 +406,7 @@ ngx_quic_tls_open(const ngx_quic_cipher_t *cipher, ngx_quic_secret_t *s, } if (EVP_DecryptUpdate(ctx, out->data, &len, in->data, - in->len - EVP_GCM_TLS_TAG_LEN) + in->len - NGX_QUIC_TAG_LEN) != 1) { EVP_CIPHER_CTX_free(ctx); @@ -415,9 +415,9 @@ ngx_quic_tls_open(const ngx_quic_cipher_t *cipher, ngx_quic_secret_t *s, } out->len = len; - tag = in->data + in->len - EVP_GCM_TLS_TAG_LEN; + tag = in->data + in->len - NGX_QUIC_TAG_LEN; - if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, EVP_GCM_TLS_TAG_LEN, tag) + if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, NGX_QUIC_TAG_LEN, tag) == 0) { EVP_CIPHER_CTX_free(ctx); @@ -519,7 +519,7 @@ ngx_quic_tls_seal(const ngx_quic_cipher_t *cipher, ngx_quic_secret_t *s, out->len += len; - if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, EVP_GCM_TLS_TAG_LEN, + if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, NGX_QUIC_TAG_LEN, out->data + in->len) == 0) { @@ -531,7 +531,7 @@ ngx_quic_tls_seal(const ngx_quic_cipher_t *cipher, ngx_quic_secret_t *s, EVP_CIPHER_CTX_free(ctx); - out->len += EVP_GCM_TLS_TAG_LEN; + out->len += NGX_QUIC_TAG_LEN; #endif return NGX_OK; } @@ -738,7 +738,7 @@ ngx_quic_create_packet(ngx_quic_header_t *pkt, ngx_str_t *res) ad.data = res->data; ad.len = ngx_quic_create_header(pkt, ad.data, &pnp); - out.len = pkt->payload.len + EVP_GCM_TLS_TAG_LEN; + out.len = pkt->payload.len + NGX_QUIC_TAG_LEN; out.data = res->data + ad.len; #ifdef NGX_QUIC_DEBUG_CRYPTO @@ -802,7 +802,7 @@ ngx_quic_create_retry_packet(ngx_quic_header_t *pkt, ngx_str_t *res) ad.len = ngx_quic_create_retry_itag(pkt, ad.data, &start); itag.data = ad.data + ad.len; - itag.len = EVP_GCM_TLS_TAG_LEN; + itag.len = NGX_QUIC_TAG_LEN; #ifdef NGX_QUIC_DEBUG_CRYPTO ngx_log_debug2(NGX_LOG_DEBUG_EVENT, pkt->log, 0, @@ -979,7 +979,7 @@ ngx_quic_decrypt(ngx_quic_header_t *pkt, uint64_t *largest_pn) * AES and ChaCha20 algorithms sample 16 bytes */ - if (len < EVP_GCM_TLS_TAG_LEN + 4) { + if (len < NGX_QUIC_TAG_LEN + 4) { return NGX_DECLINED; } @@ -1039,7 +1039,7 @@ ngx_quic_decrypt(ngx_quic_header_t *pkt, uint64_t *largest_pn) "quic ad len:%uz %xV", ad.len, &ad); #endif - pkt->payload.len = in.len - EVP_GCM_TLS_TAG_LEN; + pkt->payload.len = in.len - NGX_QUIC_TAG_LEN; pkt->payload.data = pkt->plaintext + ad.len; rc = ngx_quic_tls_open(ciphers.c, secret, &pkt->payload, diff --git a/src/event/quic/ngx_event_quic_protection.h b/src/event/quic/ngx_event_quic_protection.h index 27f8617d980..0cec1d81adc 100644 --- a/src/event/quic/ngx_event_quic_protection.h +++ b/src/event/quic/ngx_event_quic_protection.h @@ -16,8 +16,9 @@ #define NGX_QUIC_ENCRYPTION_LAST ((ssl_encryption_application) + 1) -/* RFC 5116, 5.1 and RFC 8439, 2.3 for all supported ciphers */ +/* RFC 5116, 5.1 and RFC 8439, 2.3/2.5 for all supported ciphers */ #define NGX_QUIC_IV_LEN 12 +#define NGX_QUIC_TAG_LEN 16 /* largest hash used in TLS is SHA-384 */ #define NGX_QUIC_MAX_MD_SIZE 48 diff --git a/src/event/quic/ngx_event_quic_transport.c b/src/event/quic/ngx_event_quic_transport.c index b663efbe10c..fafe85f41bd 100644 --- a/src/event/quic/ngx_event_quic_transport.c +++ b/src/event/quic/ngx_event_quic_transport.c @@ -578,7 +578,7 @@ ngx_quic_payload_size(ngx_quic_header_t *pkt, size_t pkt_len) if (ngx_quic_short_pkt(pkt->flags)) { - len = 1 + pkt->dcid.len + pkt->num_len + EVP_GCM_TLS_TAG_LEN; + len = 1 + pkt->dcid.len + pkt->num_len + NGX_QUIC_TAG_LEN; if (len > pkt_len) { return 0; } @@ -596,7 +596,7 @@ ngx_quic_payload_size(ngx_quic_header_t *pkt, size_t pkt_len) /* (pkt_len - len) is 'remainder' packet length (see RFC 9000, 17.2) */ len += ngx_quic_varint_len(pkt_len - len) - + pkt->num_len + EVP_GCM_TLS_TAG_LEN; + + pkt->num_len + NGX_QUIC_TAG_LEN; if (len > pkt_len) { return 0; @@ -622,7 +622,7 @@ ngx_quic_create_long_header(ngx_quic_header_t *pkt, u_char *out, size_t rem_len; u_char *p, *start; - rem_len = pkt->num_len + pkt->payload.len + EVP_GCM_TLS_TAG_LEN; + rem_len = pkt->num_len + pkt->payload.len + NGX_QUIC_TAG_LEN; if (out == NULL) { return 5 + 2 + pkt->dcid.len + pkt->scid.len From 3ee1051912c5e3df89d270fd5c533a4af4d0c2ee Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Fri, 9 Jun 2023 10:23:22 +0400 Subject: [PATCH 011/279] QUIC: common cipher control constants instead of GCM-related. The constants are used for both GCM and CHACHAPOLY. --- src/event/quic/ngx_event_quic_protection.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/event/quic/ngx_event_quic_protection.c b/src/event/quic/ngx_event_quic_protection.c index 9b967ab0f28..052dff97af4 100644 --- a/src/event/quic/ngx_event_quic_protection.c +++ b/src/event/quic/ngx_event_quic_protection.c @@ -384,12 +384,12 @@ ngx_quic_tls_open(const ngx_quic_cipher_t *cipher, ngx_quic_secret_t *s, return NGX_ERROR; } - if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, s->iv.len, NULL) + if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_SET_IVLEN, s->iv.len, NULL) == 0) { EVP_CIPHER_CTX_free(ctx); ngx_ssl_error(NGX_LOG_INFO, log, 0, - "EVP_CIPHER_CTX_ctrl(EVP_CTRL_GCM_SET_IVLEN) failed"); + "EVP_CIPHER_CTX_ctrl(EVP_CTRL_AEAD_SET_IVLEN) failed"); return NGX_ERROR; } @@ -417,12 +417,12 @@ ngx_quic_tls_open(const ngx_quic_cipher_t *cipher, ngx_quic_secret_t *s, out->len = len; tag = in->data + in->len - NGX_QUIC_TAG_LEN; - if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, NGX_QUIC_TAG_LEN, tag) + if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_SET_TAG, NGX_QUIC_TAG_LEN, tag) == 0) { EVP_CIPHER_CTX_free(ctx); ngx_ssl_error(NGX_LOG_INFO, log, 0, - "EVP_CIPHER_CTX_ctrl(EVP_CTRL_GCM_SET_TAG) failed"); + "EVP_CIPHER_CTX_ctrl(EVP_CTRL_AEAD_SET_TAG) failed"); return NGX_ERROR; } @@ -482,12 +482,12 @@ ngx_quic_tls_seal(const ngx_quic_cipher_t *cipher, ngx_quic_secret_t *s, return NGX_ERROR; } - if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, s->iv.len, NULL) + if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_SET_IVLEN, s->iv.len, NULL) == 0) { EVP_CIPHER_CTX_free(ctx); ngx_ssl_error(NGX_LOG_INFO, log, 0, - "EVP_CIPHER_CTX_ctrl(EVP_CTRL_GCM_SET_IVLEN) failed"); + "EVP_CIPHER_CTX_ctrl(EVP_CTRL_AEAD_SET_IVLEN) failed"); return NGX_ERROR; } @@ -519,13 +519,13 @@ ngx_quic_tls_seal(const ngx_quic_cipher_t *cipher, ngx_quic_secret_t *s, out->len += len; - if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, NGX_QUIC_TAG_LEN, + if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_GET_TAG, NGX_QUIC_TAG_LEN, out->data + in->len) == 0) { EVP_CIPHER_CTX_free(ctx); ngx_ssl_error(NGX_LOG_INFO, log, 0, - "EVP_CIPHER_CTX_ctrl(EVP_CTRL_GCM_GET_TAG) failed"); + "EVP_CIPHER_CTX_ctrl(EVP_CTRL_AEAD_GET_TAG) failed"); return NGX_ERROR; } From 69826dd4f766e5e7748f8dbdd14e6c3073250d5a Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Tue, 20 Jun 2023 16:10:49 +0400 Subject: [PATCH 012/279] QUIC: TLS_AES_128_CCM_SHA256 cipher suite support. --- src/event/quic/ngx_event_quic_protection.c | 58 ++++++++++++++++++---- src/event/quic/ngx_event_quic_protection.h | 2 +- 2 files changed, 49 insertions(+), 11 deletions(-) diff --git a/src/event/quic/ngx_event_quic_protection.c b/src/event/quic/ngx_event_quic_protection.c index 052dff97af4..e4b21fe4d6d 100644 --- a/src/event/quic/ngx_event_quic_protection.c +++ b/src/event/quic/ngx_event_quic_protection.c @@ -94,6 +94,15 @@ ngx_quic_ciphers(ngx_uint_t id, ngx_quic_ciphers_t *ciphers, len = 32; break; +#ifndef OPENSSL_IS_BORINGSSL + case TLS1_3_CK_AES_128_CCM_SHA256: + ciphers->c = EVP_aes_128_ccm(); + ciphers->hp = EVP_aes_128_ctr(); + ciphers->d = EVP_sha256(); + len = 16; + break; +#endif + default: return NGX_ERROR; } @@ -384,6 +393,17 @@ ngx_quic_tls_open(const ngx_quic_cipher_t *cipher, ngx_quic_secret_t *s, return NGX_ERROR; } + tag = in->data + in->len - NGX_QUIC_TAG_LEN; + + if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_SET_TAG, NGX_QUIC_TAG_LEN, tag) + == 0) + { + EVP_CIPHER_CTX_free(ctx); + ngx_ssl_error(NGX_LOG_INFO, log, 0, + "EVP_CIPHER_CTX_ctrl(EVP_CTRL_AEAD_SET_TAG) failed"); + return NGX_ERROR; + } + if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_SET_IVLEN, s->iv.len, NULL) == 0) { @@ -399,6 +419,15 @@ ngx_quic_tls_open(const ngx_quic_cipher_t *cipher, ngx_quic_secret_t *s, return NGX_ERROR; } + if (EVP_CIPHER_mode(cipher) == EVP_CIPH_CCM_MODE + && EVP_DecryptUpdate(ctx, NULL, &len, NULL, in->len - NGX_QUIC_TAG_LEN) + != 1) + { + EVP_CIPHER_CTX_free(ctx); + ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_DecryptUpdate() failed"); + return NGX_ERROR; + } + if (EVP_DecryptUpdate(ctx, NULL, &len, ad->data, ad->len) != 1) { EVP_CIPHER_CTX_free(ctx); ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_DecryptUpdate() failed"); @@ -415,16 +444,6 @@ ngx_quic_tls_open(const ngx_quic_cipher_t *cipher, ngx_quic_secret_t *s, } out->len = len; - tag = in->data + in->len - NGX_QUIC_TAG_LEN; - - if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_SET_TAG, NGX_QUIC_TAG_LEN, tag) - == 0) - { - EVP_CIPHER_CTX_free(ctx); - ngx_ssl_error(NGX_LOG_INFO, log, 0, - "EVP_CIPHER_CTX_ctrl(EVP_CTRL_AEAD_SET_TAG) failed"); - return NGX_ERROR; - } if (EVP_DecryptFinal_ex(ctx, out->data + len, &len) <= 0) { EVP_CIPHER_CTX_free(ctx); @@ -482,6 +501,17 @@ ngx_quic_tls_seal(const ngx_quic_cipher_t *cipher, ngx_quic_secret_t *s, return NGX_ERROR; } + if (EVP_CIPHER_mode(cipher) == EVP_CIPH_CCM_MODE + && EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_SET_TAG, NGX_QUIC_TAG_LEN, + NULL) + == 0) + { + EVP_CIPHER_CTX_free(ctx); + ngx_ssl_error(NGX_LOG_INFO, log, 0, + "EVP_CIPHER_CTX_ctrl(EVP_CTRL_AEAD_SET_TAG) failed"); + return NGX_ERROR; + } + if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_SET_IVLEN, s->iv.len, NULL) == 0) { @@ -497,6 +527,14 @@ ngx_quic_tls_seal(const ngx_quic_cipher_t *cipher, ngx_quic_secret_t *s, return NGX_ERROR; } + if (EVP_CIPHER_mode(cipher) == EVP_CIPH_CCM_MODE + && EVP_EncryptUpdate(ctx, NULL, &len, NULL, in->len) != 1) + { + EVP_CIPHER_CTX_free(ctx); + ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_EncryptUpdate() failed"); + return NGX_ERROR; + } + if (EVP_EncryptUpdate(ctx, NULL, &len, ad->data, ad->len) != 1) { EVP_CIPHER_CTX_free(ctx); ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_EncryptUpdate() failed"); diff --git a/src/event/quic/ngx_event_quic_protection.h b/src/event/quic/ngx_event_quic_protection.h index 0cec1d81adc..4e56ea9d1db 100644 --- a/src/event/quic/ngx_event_quic_protection.h +++ b/src/event/quic/ngx_event_quic_protection.h @@ -16,7 +16,7 @@ #define NGX_QUIC_ENCRYPTION_LAST ((ssl_encryption_application) + 1) -/* RFC 5116, 5.1 and RFC 8439, 2.3/2.5 for all supported ciphers */ +/* RFC 5116, 5.1/5.3 and RFC 8439, 2.3/2.5 for all supported ciphers */ #define NGX_QUIC_IV_LEN 12 #define NGX_QUIC_TAG_LEN 16 From 68690b034580f6ec04fafdbbcb6a4e817f8a330b Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Tue, 20 Jun 2023 17:59:01 +0400 Subject: [PATCH 013/279] QUIC: unified ngx_quic_tls_open() and ngx_quic_tls_seal(). --- src/event/quic/ngx_event_quic_protection.c | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/src/event/quic/ngx_event_quic_protection.c b/src/event/quic/ngx_event_quic_protection.c index e4b21fe4d6d..9aa1c3580c4 100644 --- a/src/event/quic/ngx_event_quic_protection.c +++ b/src/event/quic/ngx_event_quic_protection.c @@ -378,7 +378,6 @@ ngx_quic_tls_open(const ngx_quic_cipher_t *cipher, ngx_quic_secret_t *s, EVP_AEAD_CTX_free(ctx); #else int len; - u_char *tag; EVP_CIPHER_CTX *ctx; ctx = EVP_CIPHER_CTX_new(); @@ -393,9 +392,10 @@ ngx_quic_tls_open(const ngx_quic_cipher_t *cipher, ngx_quic_secret_t *s, return NGX_ERROR; } - tag = in->data + in->len - NGX_QUIC_TAG_LEN; + in->len -= NGX_QUIC_TAG_LEN; - if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_SET_TAG, NGX_QUIC_TAG_LEN, tag) + if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_SET_TAG, NGX_QUIC_TAG_LEN, + in->data + in->len) == 0) { EVP_CIPHER_CTX_free(ctx); @@ -420,8 +420,7 @@ ngx_quic_tls_open(const ngx_quic_cipher_t *cipher, ngx_quic_secret_t *s, } if (EVP_CIPHER_mode(cipher) == EVP_CIPH_CCM_MODE - && EVP_DecryptUpdate(ctx, NULL, &len, NULL, in->len - NGX_QUIC_TAG_LEN) - != 1) + && EVP_DecryptUpdate(ctx, NULL, &len, NULL, in->len) != 1) { EVP_CIPHER_CTX_free(ctx); ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_DecryptUpdate() failed"); @@ -434,10 +433,7 @@ ngx_quic_tls_open(const ngx_quic_cipher_t *cipher, ngx_quic_secret_t *s, return NGX_ERROR; } - if (EVP_DecryptUpdate(ctx, out->data, &len, in->data, - in->len - NGX_QUIC_TAG_LEN) - != 1) - { + if (EVP_DecryptUpdate(ctx, out->data, &len, in->data, in->len) != 1) { EVP_CIPHER_CTX_free(ctx); ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_DecryptUpdate() failed"); return NGX_ERROR; @@ -445,7 +441,7 @@ ngx_quic_tls_open(const ngx_quic_cipher_t *cipher, ngx_quic_secret_t *s, out->len = len; - if (EVP_DecryptFinal_ex(ctx, out->data + len, &len) <= 0) { + if (EVP_DecryptFinal_ex(ctx, out->data + out->len, &len) <= 0) { EVP_CIPHER_CTX_free(ctx); ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_DecryptFinal_ex failed"); return NGX_ERROR; @@ -558,7 +554,7 @@ ngx_quic_tls_seal(const ngx_quic_cipher_t *cipher, ngx_quic_secret_t *s, out->len += len; if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_GET_TAG, NGX_QUIC_TAG_LEN, - out->data + in->len) + out->data + out->len) == 0) { EVP_CIPHER_CTX_free(ctx); From dc9017ee482ef2ffad822942f78c07ee210b205a Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Tue, 20 Jun 2023 17:59:02 +0400 Subject: [PATCH 014/279] QUIC: style. --- src/event/quic/ngx_event_quic_protection.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/event/quic/ngx_event_quic_protection.c b/src/event/quic/ngx_event_quic_protection.c index 9aa1c3580c4..9e731e3917e 100644 --- a/src/event/quic/ngx_event_quic_protection.c +++ b/src/event/quic/ngx_event_quic_protection.c @@ -352,8 +352,7 @@ ngx_hkdf_extract(u_char *out_key, size_t *out_len, const EVP_MD *digest, static ngx_int_t ngx_quic_tls_open(const ngx_quic_cipher_t *cipher, ngx_quic_secret_t *s, - ngx_str_t *out, u_char *nonce, ngx_str_t *in, ngx_str_t *ad, - ngx_log_t *log) + ngx_str_t *out, u_char *nonce, ngx_str_t *in, ngx_str_t *ad, ngx_log_t *log) { #ifdef OPENSSL_IS_BORINGSSL @@ -563,10 +562,11 @@ ngx_quic_tls_seal(const ngx_quic_cipher_t *cipher, ngx_quic_secret_t *s, return NGX_ERROR; } - EVP_CIPHER_CTX_free(ctx); - out->len += NGX_QUIC_TAG_LEN; + + EVP_CIPHER_CTX_free(ctx); #endif + return NGX_OK; } From b051754d9d6548e870ce6085d13402c9a3e06d3c Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Fri, 16 Jun 2023 17:13:29 +0400 Subject: [PATCH 015/279] QUIC: removed TLS1_3_CK_* macros wrap up. They were preserved in 172705615d04 to ease transition from older BoringSSL. --- src/event/quic/ngx_event_quic_protection.c | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/event/quic/ngx_event_quic_protection.c b/src/event/quic/ngx_event_quic_protection.c index 9e731e3917e..ecac6f57876 100644 --- a/src/event/quic/ngx_event_quic_protection.c +++ b/src/event/quic/ngx_event_quic_protection.c @@ -15,13 +15,6 @@ #define NGX_QUIC_AES_128_KEY_LEN 16 -#ifndef TLS1_3_CK_AES_128_GCM_SHA256 -#define TLS1_3_CK_AES_128_GCM_SHA256 0x03001301 -#define TLS1_3_CK_AES_256_GCM_SHA384 0x03001302 -#define TLS1_3_CK_CHACHA20_POLY1305_SHA256 \ - 0x03001303 -#endif - static ngx_int_t ngx_hkdf_expand(u_char *out_key, size_t out_len, const EVP_MD *digest, const u_char *prk, size_t prk_len, From 6bdfd58f2660c394306ef34fe825d2f02f5ba813 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Thu, 8 Jun 2023 14:58:01 +0400 Subject: [PATCH 016/279] QUIC: use AEAD to encrypt address validation tokens. Previously used AES256-CBC is now substituted with AES256-GCM. Although there seem to be no tangible consequences of token integrity loss. --- src/event/quic/ngx_event_quic_tokens.c | 44 +++++++++++++++++++------- src/event/quic/ngx_event_quic_tokens.h | 9 +++--- 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/src/event/quic/ngx_event_quic_tokens.c b/src/event/quic/ngx_event_quic_tokens.c index e67825aabaa..c1da0d47260 100644 --- a/src/event/quic/ngx_event_quic_tokens.c +++ b/src/event/quic/ngx_event_quic_tokens.c @@ -69,11 +69,10 @@ ngx_quic_new_token(ngx_log_t *log, struct sockaddr *sockaddr, len = p - in; - cipher = EVP_aes_256_cbc(); - iv_len = NGX_QUIC_AES_256_CBC_IV_LEN; + cipher = EVP_aes_256_gcm(); + iv_len = NGX_QUIC_AES_256_GCM_IV_LEN; - if ((size_t) (iv_len + len + NGX_QUIC_AES_256_CBC_BLOCK_SIZE) > token->len) - { + if ((size_t) (iv_len + len + NGX_QUIC_AES_256_GCM_TAG_LEN) > token->len) { ngx_log_error(NGX_LOG_ALERT, log, 0, "quic token buffer is too small"); return NGX_ERROR; } @@ -108,6 +107,17 @@ ngx_quic_new_token(ngx_log_t *log, struct sockaddr *sockaddr, token->len += len; + if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_GET_TAG, + NGX_QUIC_AES_256_GCM_TAG_LEN, + token->data + token->len) + == 0) + { + EVP_CIPHER_CTX_free(ctx); + return NGX_ERROR; + } + + token->len += NGX_QUIC_AES_256_GCM_TAG_LEN; + EVP_CIPHER_CTX_free(ctx); #ifdef NGX_QUIC_DEBUG_PACKETS @@ -184,17 +194,19 @@ ngx_quic_validate_token(ngx_connection_t *c, u_char *key, /* Retry token or NEW_TOKEN in a previous connection */ - cipher = EVP_aes_256_cbc(); + cipher = EVP_aes_256_gcm(); iv = pkt->token.data; - iv_len = NGX_QUIC_AES_256_CBC_IV_LEN; + iv_len = NGX_QUIC_AES_256_GCM_IV_LEN; /* sanity checks */ - if (pkt->token.len < (size_t) iv_len + NGX_QUIC_AES_256_CBC_BLOCK_SIZE) { + if (pkt->token.len < (size_t) iv_len + NGX_QUIC_AES_256_GCM_TAG_LEN) { goto garbage; } - if (pkt->token.len > (size_t) iv_len + NGX_QUIC_MAX_TOKEN_SIZE) { + if (pkt->token.len > (size_t) iv_len + NGX_QUIC_MAX_TOKEN_SIZE + + NGX_QUIC_AES_256_GCM_TAG_LEN) + { goto garbage; } @@ -209,15 +221,23 @@ ngx_quic_validate_token(ngx_connection_t *c, u_char *key, } p = pkt->token.data + iv_len; - len = pkt->token.len - iv_len; + len = pkt->token.len - iv_len - NGX_QUIC_AES_256_GCM_TAG_LEN; + + if (EVP_DecryptUpdate(ctx, tdec, &tlen, p, len) != 1) { + EVP_CIPHER_CTX_free(ctx); + goto garbage; + } + total = tlen; - if (EVP_DecryptUpdate(ctx, tdec, &len, p, len) != 1) { + if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_SET_TAG, + NGX_QUIC_AES_256_GCM_TAG_LEN, p + len) + == 0) + { EVP_CIPHER_CTX_free(ctx); goto garbage; } - total = len; - if (EVP_DecryptFinal_ex(ctx, tdec + len, &tlen) <= 0) { + if (EVP_DecryptFinal_ex(ctx, tdec + tlen, &tlen) <= 0) { EVP_CIPHER_CTX_free(ctx); goto garbage; } diff --git a/src/event/quic/ngx_event_quic_tokens.h b/src/event/quic/ngx_event_quic_tokens.h index 4a40f7b3aa4..ee3fe5b9a00 100644 --- a/src/event/quic/ngx_event_quic_tokens.h +++ b/src/event/quic/ngx_event_quic_tokens.h @@ -15,13 +15,12 @@ #define NGX_QUIC_MAX_TOKEN_SIZE 64 /* SHA-1(addr)=20 + sizeof(time_t) + retry(1) + odcid.len(1) + odcid */ -/* RFC 3602, 2.1 and 2.4 for AES-CBC block size and IV length */ -#define NGX_QUIC_AES_256_CBC_IV_LEN 16 -#define NGX_QUIC_AES_256_CBC_BLOCK_SIZE 16 +#define NGX_QUIC_AES_256_GCM_IV_LEN 12 +#define NGX_QUIC_AES_256_GCM_TAG_LEN 16 -#define NGX_QUIC_TOKEN_BUF_SIZE (NGX_QUIC_AES_256_CBC_IV_LEN \ +#define NGX_QUIC_TOKEN_BUF_SIZE (NGX_QUIC_AES_256_GCM_IV_LEN \ + NGX_QUIC_MAX_TOKEN_SIZE \ - + NGX_QUIC_AES_256_CBC_BLOCK_SIZE) + + NGX_QUIC_AES_256_GCM_TAG_LEN) ngx_int_t ngx_quic_new_sr_token(ngx_connection_t *c, ngx_str_t *cid, From 4d3a9cc11fb883a7fa8b4f1859047acc121741d3 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Wed, 12 Jul 2023 15:27:35 +0400 Subject: [PATCH 017/279] HTTP/3: fixed $body_bytes_sent. --- src/http/v3/ngx_http_v3_filter_module.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/http/v3/ngx_http_v3_filter_module.c b/src/http/v3/ngx_http_v3_filter_module.c index 7addf275d16..4d2276dc096 100644 --- a/src/http/v3/ngx_http_v3_filter_module.c +++ b/src/http/v3/ngx_http_v3_filter_module.c @@ -581,6 +581,7 @@ ngx_http_v3_header_filter(ngx_http_request_t *r) for (cl = out; cl; cl = cl->next) { h3c->total_bytes += cl->buf->last - cl->buf->pos; + r->header_size += cl->buf->last - cl->buf->pos; } return ngx_http_write_filter(r, out); From 9e1a000f2bb5d32d0fa8ff804ce8a0f7539694d7 Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Wed, 19 Jul 2023 05:09:23 +0300 Subject: [PATCH 018/279] Core: fixed environment variables on exit. Similarly to 6822:c045b4926b2c, environment variables introduced with the "env" directive (and "NGINX_BPF_MAPS" added by QUIC) are now allocated via ngx_alloc(), and explicitly freed by a cleanup handler if no longer used. In collaboration with Sergey Kandaurov. --- src/core/nginx.c | 53 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/src/core/nginx.c b/src/core/nginx.c index 062ab08981b..0deb27b7f98 100644 --- a/src/core/nginx.c +++ b/src/core/nginx.c @@ -13,6 +13,7 @@ static void ngx_show_version_info(void); static ngx_int_t ngx_add_inherited_sockets(ngx_cycle_t *cycle); static void ngx_cleanup_environment(void *data); +static void ngx_cleanup_environment_variable(void *data); static ngx_int_t ngx_get_options(int argc, char *const *argv); static ngx_int_t ngx_process_options(ngx_cycle_t *cycle); static ngx_int_t ngx_save_argv(ngx_cycle_t *cycle, int argc, char *const *argv); @@ -518,7 +519,8 @@ ngx_add_inherited_sockets(ngx_cycle_t *cycle) char ** ngx_set_environment(ngx_cycle_t *cycle, ngx_uint_t *last) { - char **p, **env; + char **p, **env, *str; + size_t len; ngx_str_t *var; ngx_uint_t i, n; ngx_core_conf_t *ccf; @@ -600,7 +602,31 @@ ngx_set_environment(ngx_cycle_t *cycle, ngx_uint_t *last) for (i = 0; i < ccf->env.nelts; i++) { if (var[i].data[var[i].len] == '=') { - env[n++] = (char *) var[i].data; + + if (last) { + env[n++] = (char *) var[i].data; + continue; + } + + cln = ngx_pool_cleanup_add(cycle->pool, 0); + if (cln == NULL) { + return NULL; + } + + len = ngx_strlen(var[i].data) + 1; + + str = ngx_alloc(len, cycle->log); + if (str == NULL) { + return NULL; + } + + ngx_memcpy(str, var[i].data, len); + + cln->handler = ngx_cleanup_environment_variable; + cln->data = str; + + env[n++] = str; + continue; } @@ -645,6 +671,29 @@ ngx_cleanup_environment(void *data) } +static void +ngx_cleanup_environment_variable(void *data) +{ + char *var = data; + + char **p; + + for (p = environ; *p; p++) { + + /* + * if an environment variable is still used, as it happens on exit, + * the only option is to leak it + */ + + if (*p == var) { + return; + } + } + + ngx_free(var); +} + + ngx_pid_t ngx_exec_new_binary(ngx_cycle_t *cycle, char *const *argv) { From 1c6183725247024f1bca73ac9a833098af7558af Mon Sep 17 00:00:00 2001 From: Gena Makhomed Date: Mon, 24 Jul 2023 18:04:41 +0300 Subject: [PATCH 019/279] Contrib: vim syntax, update core and 3rd party module directives. List of 3rd party modules github repositories was obtained from https://github.com/freebsd/freebsd-ports/blob/main/www/nginx-devel/Makefile.extmod --- contrib/vim/syntax/nginx.vim | 217 +++++++++++++++-------------------- 1 file changed, 93 insertions(+), 124 deletions(-) diff --git a/contrib/vim/syntax/nginx.vim b/contrib/vim/syntax/nginx.vim index 7d587fc48e1..29eef7a23cd 100644 --- a/contrib/vim/syntax/nginx.vim +++ b/contrib/vim/syntax/nginx.vim @@ -65,12 +65,12 @@ syn match ngxListenComment '#.*$' \ contained \ nextgroup=@ngxListenParams skipwhite skipempty syn keyword ngxListenOptions contained - \ default_server ssl http2 proxy_protocol + \ default_server ssl quic proxy_protocol \ setfib fastopen backlog rcvbuf sndbuf accept_filter deferred bind \ ipv6only reuseport so_keepalive \ nextgroup=@ngxListenParams skipwhite skipempty syn keyword ngxListenOptionsDeprecated contained - \ spdy + \ http2 \ nextgroup=@ngxListenParams skipwhite skipempty syn cluster ngxListenParams \ contains=ngxListenParam,ngxListenString,ngxListenComment @@ -90,7 +90,6 @@ syn keyword ngxDirectiveBlock contained if syn keyword ngxDirectiveBlock contained geo syn keyword ngxDirectiveBlock contained map syn keyword ngxDirectiveBlock contained split_clients -syn keyword ngxDirectiveBlock contained match syn keyword ngxDirectiveImportant contained include syn keyword ngxDirectiveImportant contained root @@ -113,7 +112,6 @@ syn keyword ngxDirectiveError contained post_action syn keyword ngxDirectiveDeprecated contained proxy_downstream_buffer syn keyword ngxDirectiveDeprecated contained proxy_upstream_buffer -syn keyword ngxDirectiveDeprecated contained ssl syn keyword ngxDirectiveDeprecated contained http2_idle_timeout syn keyword ngxDirectiveDeprecated contained http2_max_field_size syn keyword ngxDirectiveDeprecated contained http2_max_header_size @@ -136,7 +134,6 @@ syn keyword ngxDirective contained alias syn keyword ngxDirective contained allow syn keyword ngxDirective contained ancient_browser syn keyword ngxDirective contained ancient_browser_value -syn keyword ngxDirective contained api syn keyword ngxDirective contained auth_basic syn keyword ngxDirective contained auth_basic_user_file syn keyword ngxDirective contained auth_delay @@ -144,15 +141,6 @@ syn keyword ngxDirective contained auth_http syn keyword ngxDirective contained auth_http_header syn keyword ngxDirective contained auth_http_pass_client_cert syn keyword ngxDirective contained auth_http_timeout -syn keyword ngxDirective contained auth_jwt -syn keyword ngxDirective contained auth_jwt_claim_set -syn keyword ngxDirective contained auth_jwt_header_set -syn keyword ngxDirective contained auth_jwt_key_cache -syn keyword ngxDirective contained auth_jwt_key_file -syn keyword ngxDirective contained auth_jwt_key_request -syn keyword ngxDirective contained auth_jwt_leeway -syn keyword ngxDirective contained auth_jwt_require -syn keyword ngxDirective contained auth_jwt_type syn keyword ngxDirective contained auth_request syn keyword ngxDirective contained auth_request_set syn keyword ngxDirective contained autoindex @@ -193,8 +181,6 @@ syn keyword ngxDirective contained error_log syn keyword ngxDirective contained etag syn keyword ngxDirective contained eventport_events syn keyword ngxDirective contained expires -syn keyword ngxDirective contained f4f -syn keyword ngxDirective contained f4f_buffer_size syn keyword ngxDirective contained fastcgi_bind syn keyword ngxDirective contained fastcgi_buffer_size syn keyword ngxDirective contained fastcgi_buffering @@ -211,7 +197,6 @@ syn keyword ngxDirective contained fastcgi_cache_max_range_offset syn keyword ngxDirective contained fastcgi_cache_methods syn keyword ngxDirective contained fastcgi_cache_min_uses syn keyword ngxDirective contained fastcgi_cache_path -syn keyword ngxDirective contained fastcgi_cache_purge syn keyword ngxDirective contained fastcgi_cache_revalidate syn keyword ngxDirective contained fastcgi_cache_use_stale syn keyword ngxDirective contained fastcgi_cache_valid @@ -295,14 +280,7 @@ syn keyword ngxDirective contained gzip_types syn keyword ngxDirective contained gzip_vary syn keyword ngxDirective contained gzip_window syn keyword ngxDirective contained hash -syn keyword ngxDirective contained health_check -syn keyword ngxDirective contained health_check_timeout -syn keyword ngxDirective contained hls -syn keyword ngxDirective contained hls_buffers -syn keyword ngxDirective contained hls_forward_args -syn keyword ngxDirective contained hls_fragment -syn keyword ngxDirective contained hls_mp4_buffer_size -syn keyword ngxDirective contained hls_mp4_max_buffer_size +syn keyword ngxDirective contained http2 syn keyword ngxDirective contained http2_body_preread_size syn keyword ngxDirective contained http2_chunk_size syn keyword ngxDirective contained http2_max_concurrent_pushes @@ -312,6 +290,10 @@ syn keyword ngxDirective contained http2_push syn keyword ngxDirective contained http2_push_preload syn keyword ngxDirective contained http2_recv_buffer_size syn keyword ngxDirective contained http2_streams_index_size +syn keyword ngxDirective contained http3 +syn keyword ngxDirective contained http3_hq +syn keyword ngxDirective contained http3_max_concurrent_streams +syn keyword ngxDirective contained http3_stream_buffer_size syn keyword ngxDirective contained if_modified_since syn keyword ngxDirective contained ignore_invalid_headers syn keyword ngxDirective contained image_filter @@ -342,21 +324,20 @@ syn keyword ngxDirective contained js_filter syn keyword ngxDirective contained js_header_filter syn keyword ngxDirective contained js_import syn keyword ngxDirective contained js_path +syn keyword ngxDirective contained js_preload_object syn keyword ngxDirective contained js_preread syn keyword ngxDirective contained js_set +syn keyword ngxDirective contained js_shared_dict_zone syn keyword ngxDirective contained js_var syn keyword ngxDirective contained keepalive syn keyword ngxDirective contained keepalive_disable syn keyword ngxDirective contained keepalive_requests syn keyword ngxDirective contained keepalive_time syn keyword ngxDirective contained keepalive_timeout -syn keyword ngxDirective contained keyval -syn keyword ngxDirective contained keyval_zone syn keyword ngxDirective contained kqueue_changes syn keyword ngxDirective contained kqueue_events syn keyword ngxDirective contained large_client_header_buffers syn keyword ngxDirective contained least_conn -syn keyword ngxDirective contained least_time syn keyword ngxDirective contained limit_conn syn keyword ngxDirective contained limit_conn_dry_run syn keyword ngxDirective contained limit_conn_log_level @@ -400,14 +381,11 @@ syn keyword ngxDirective contained modern_browser syn keyword ngxDirective contained modern_browser_value syn keyword ngxDirective contained mp4 syn keyword ngxDirective contained mp4_buffer_size -syn keyword ngxDirective contained mp4_limit_rate -syn keyword ngxDirective contained mp4_limit_rate_after syn keyword ngxDirective contained mp4_max_buffer_size syn keyword ngxDirective contained mp4_start_key_frame syn keyword ngxDirective contained msie_padding syn keyword ngxDirective contained msie_refresh syn keyword ngxDirective contained multi_accept -syn keyword ngxDirective contained ntlm syn keyword ngxDirective contained open_file_cache syn keyword ngxDirective contained open_file_cache_errors syn keyword ngxDirective contained open_file_cache_events @@ -450,7 +428,6 @@ syn keyword ngxDirective contained proxy_cache_max_range_offset syn keyword ngxDirective contained proxy_cache_methods syn keyword ngxDirective contained proxy_cache_min_uses syn keyword ngxDirective contained proxy_cache_path -syn keyword ngxDirective contained proxy_cache_purge syn keyword ngxDirective contained proxy_cache_revalidate syn keyword ngxDirective contained proxy_cache_use_stale syn keyword ngxDirective contained proxy_cache_valid @@ -488,7 +465,6 @@ syn keyword ngxDirective contained proxy_requests syn keyword ngxDirective contained proxy_responses syn keyword ngxDirective contained proxy_send_lowat syn keyword ngxDirective contained proxy_send_timeout -syn keyword ngxDirective contained proxy_session_drop syn keyword ngxDirective contained proxy_set_body syn keyword ngxDirective contained proxy_set_header syn keyword ngxDirective contained proxy_smtp_auth @@ -513,7 +489,11 @@ syn keyword ngxDirective contained proxy_temp_file_write_size syn keyword ngxDirective contained proxy_temp_path syn keyword ngxDirective contained proxy_timeout syn keyword ngxDirective contained proxy_upload_rate -syn keyword ngxDirective contained queue +syn keyword ngxDirective contained quic_active_connection_id_limit +syn keyword ngxDirective contained quic_bpf +syn keyword ngxDirective contained quic_gso +syn keyword ngxDirective contained quic_host_key +syn keyword ngxDirective contained quic_retry syn keyword ngxDirective contained random syn keyword ngxDirective contained random_index syn keyword ngxDirective contained read_ahead @@ -544,7 +524,6 @@ syn keyword ngxDirective contained scgi_cache_max_range_offset syn keyword ngxDirective contained scgi_cache_methods syn keyword ngxDirective contained scgi_cache_min_uses syn keyword ngxDirective contained scgi_cache_path -syn keyword ngxDirective contained scgi_cache_purge syn keyword ngxDirective contained scgi_cache_revalidate syn keyword ngxDirective contained scgi_cache_use_stale syn keyword ngxDirective contained scgi_cache_valid @@ -583,9 +562,6 @@ syn keyword ngxDirective contained server_name_in_redirect syn keyword ngxDirective contained server_names_hash_bucket_size syn keyword ngxDirective contained server_names_hash_max_size syn keyword ngxDirective contained server_tokens -syn keyword ngxDirective contained session_log -syn keyword ngxDirective contained session_log_format -syn keyword ngxDirective contained session_log_zone syn keyword ngxDirective contained set_real_ip_from syn keyword ngxDirective contained slice syn keyword ngxDirective contained smtp_auth @@ -633,11 +609,6 @@ syn keyword ngxDirective contained ssl_trusted_certificate syn keyword ngxDirective contained ssl_verify_client syn keyword ngxDirective contained ssl_verify_depth syn keyword ngxDirective contained starttls -syn keyword ngxDirective contained state -syn keyword ngxDirective contained status -syn keyword ngxDirective contained status_format -syn keyword ngxDirective contained status_zone -syn keyword ngxDirective contained sticky syn keyword ngxDirective contained stub_status syn keyword ngxDirective contained sub_filter syn keyword ngxDirective contained sub_filter_last_modified @@ -680,7 +651,6 @@ syn keyword ngxDirective contained uwsgi_cache_max_range_offset syn keyword ngxDirective contained uwsgi_cache_methods syn keyword ngxDirective contained uwsgi_cache_min_uses syn keyword ngxDirective contained uwsgi_cache_path -syn keyword ngxDirective contained uwsgi_cache_purge syn keyword ngxDirective contained uwsgi_cache_revalidate syn keyword ngxDirective contained uwsgi_cache_use_stale syn keyword ngxDirective contained uwsgi_cache_valid @@ -744,6 +714,62 @@ syn keyword ngxDirective contained xslt_string_param syn keyword ngxDirective contained xslt_stylesheet syn keyword ngxDirective contained xslt_types syn keyword ngxDirective contained zone + +" nginx-plus commercial extensions directives + +syn keyword ngxDirectiveBlock contained match +syn keyword ngxDirectiveBlock contained otel_exporter + +syn keyword ngxDirective contained api +syn keyword ngxDirective contained auth_jwt +syn keyword ngxDirective contained auth_jwt_claim_set +syn keyword ngxDirective contained auth_jwt_header_set +syn keyword ngxDirective contained auth_jwt_key_cache +syn keyword ngxDirective contained auth_jwt_key_file +syn keyword ngxDirective contained auth_jwt_key_request +syn keyword ngxDirective contained auth_jwt_leeway +syn keyword ngxDirective contained auth_jwt_require +syn keyword ngxDirective contained auth_jwt_type +syn keyword ngxDirective contained f4f +syn keyword ngxDirective contained f4f_buffer_size +syn keyword ngxDirective contained fastcgi_cache_purge +syn keyword ngxDirective contained health_check +syn keyword ngxDirective contained health_check_timeout +syn keyword ngxDirective contained hls +syn keyword ngxDirective contained hls_buffers +syn keyword ngxDirective contained hls_forward_args +syn keyword ngxDirective contained hls_fragment +syn keyword ngxDirective contained hls_mp4_buffer_size +syn keyword ngxDirective contained hls_mp4_max_buffer_size +syn keyword ngxDirective contained internal_redirect +syn keyword ngxDirective contained keyval +syn keyword ngxDirective contained keyval_zone +syn keyword ngxDirective contained least_time +syn keyword ngxDirective contained mp4_limit_rate +syn keyword ngxDirective contained mp4_limit_rate_after +syn keyword ngxDirective contained mqtt +syn keyword ngxDirective contained mqtt_preread +syn keyword ngxDirective contained mqtt_rewrite_buffer_size +syn keyword ngxDirective contained mqtt_set_connect +syn keyword ngxDirective contained ntlm +syn keyword ngxDirective contained otel_service_name +syn keyword ngxDirective contained otel_span_attr +syn keyword ngxDirective contained otel_span_name +syn keyword ngxDirective contained otel_trace +syn keyword ngxDirective contained otel_trace_context +syn keyword ngxDirective contained proxy_cache_purge +syn keyword ngxDirective contained proxy_session_drop +syn keyword ngxDirective contained queue +syn keyword ngxDirective contained scgi_cache_purge +syn keyword ngxDirective contained session_log +syn keyword ngxDirective contained session_log_format +syn keyword ngxDirective contained session_log_zone +syn keyword ngxDirective contained state +syn keyword ngxDirective contained status +syn keyword ngxDirective contained status_format +syn keyword ngxDirective contained status_zone +syn keyword ngxDirective contained sticky +syn keyword ngxDirective contained uwsgi_cache_purge syn keyword ngxDirective contained zone_sync syn keyword ngxDirective contained zone_sync_buffers syn keyword ngxDirective contained zone_sync_connect_retry_interval @@ -766,7 +792,6 @@ syn keyword ngxDirective contained zone_sync_ssl_verify syn keyword ngxDirective contained zone_sync_ssl_verify_depth syn keyword ngxDirective contained zone_sync_timeout - " 3rd party modules list taken from " https://github.com/freebsd/freebsd-ports/blob/main/www/nginx-devel/Makefile.extmod " ---------------------------------------------------------------------------------- @@ -837,52 +862,6 @@ syn keyword ngxDirectiveThirdParty contained brotli_window " https://github.com/torden/ngx_cache_purge syn keyword ngxDirectiveThirdParty contained cache_purge_response_type -" https://github.com/nginx-clojure/nginx-clojure -syn keyword ngxDirectiveThirdParty contained access_handler_code -syn keyword ngxDirectiveThirdParty contained access_handler_name -syn keyword ngxDirectiveThirdParty contained access_handler_property -syn keyword ngxDirectiveThirdParty contained access_handler_type -syn keyword ngxDirectiveThirdParty contained always_read_body -syn keyword ngxDirectiveThirdParty contained auto_upgrade_ws -syn keyword ngxDirectiveThirdParty contained body_filter_code -syn keyword ngxDirectiveThirdParty contained body_filter_name -syn keyword ngxDirectiveThirdParty contained body_filter_property -syn keyword ngxDirectiveThirdParty contained body_filter_type -syn keyword ngxDirectiveThirdParty contained content_handler_code -syn keyword ngxDirectiveThirdParty contained content_handler_name -syn keyword ngxDirectiveThirdParty contained content_handler_property -syn keyword ngxDirectiveThirdParty contained content_handler_type -syn keyword ngxDirectiveThirdParty contained handler_code -syn keyword ngxDirectiveThirdParty contained handler_name -syn keyword ngxDirectiveThirdParty contained handler_type -syn keyword ngxDirectiveThirdParty contained handlers_lazy_init -syn keyword ngxDirectiveThirdParty contained header_filter_code -syn keyword ngxDirectiveThirdParty contained header_filter_name -syn keyword ngxDirectiveThirdParty contained header_filter_property -syn keyword ngxDirectiveThirdParty contained header_filter_type -syn keyword ngxDirectiveThirdParty contained jvm_classpath -syn keyword ngxDirectiveThirdParty contained jvm_classpath_check -syn keyword ngxDirectiveThirdParty contained jvm_exit_handler_code -syn keyword ngxDirectiveThirdParty contained jvm_exit_handler_name -syn keyword ngxDirectiveThirdParty contained jvm_handler_type -syn keyword ngxDirectiveThirdParty contained jvm_init_handler_code -syn keyword ngxDirectiveThirdParty contained jvm_init_handler_name -syn keyword ngxDirectiveThirdParty contained jvm_options -syn keyword ngxDirectiveThirdParty contained jvm_path -syn keyword ngxDirectiveThirdParty contained jvm_var -syn keyword ngxDirectiveThirdParty contained jvm_workers -syn keyword ngxDirectiveThirdParty contained log_handler_code -syn keyword ngxDirectiveThirdParty contained log_handler_name -syn keyword ngxDirectiveThirdParty contained log_handler_property -syn keyword ngxDirectiveThirdParty contained log_handler_type -syn keyword ngxDirectiveThirdParty contained max_balanced_tcp_connections -syn keyword ngxDirectiveThirdParty contained rewrite_handler_code -syn keyword ngxDirectiveThirdParty contained rewrite_handler_name -syn keyword ngxDirectiveThirdParty contained rewrite_handler_property -syn keyword ngxDirectiveThirdParty contained rewrite_handler_type -syn keyword ngxDirectiveThirdParty contained shared_map -syn keyword ngxDirectiveThirdParty contained write_page_size - " https://github.com/AirisX/nginx_cookie_flag_module syn keyword ngxDirectiveThirdParty contained set_cookie_flag @@ -932,29 +911,6 @@ syn keyword ngxDirectiveThirdParty contained dns_update syn keyword ngxDirectiveThirdParty contained dynamic_state_file syn keyword ngxDirectiveThirdParty contained dynamic_upstream -" https://github.com/ZigzagAK/ngx_dynamic_healthcheck -syn keyword ngxDirectiveThirdParty contained check -syn keyword ngxDirectiveThirdParty contained check_disable_host -syn keyword ngxDirectiveThirdParty contained check_exclude_host -syn keyword ngxDirectiveThirdParty contained check_persistent -syn keyword ngxDirectiveThirdParty contained check_request_body -syn keyword ngxDirectiveThirdParty contained check_request_headers -syn keyword ngxDirectiveThirdParty contained check_request_uri -syn keyword ngxDirectiveThirdParty contained check_response_body -syn keyword ngxDirectiveThirdParty contained check_response_codes -syn keyword ngxDirectiveThirdParty contained healthcheck -syn keyword ngxDirectiveThirdParty contained healthcheck_buffer_size -syn keyword ngxDirectiveThirdParty contained healthcheck_disable_host -syn keyword ngxDirectiveThirdParty contained healthcheck_get -syn keyword ngxDirectiveThirdParty contained healthcheck_persistent -syn keyword ngxDirectiveThirdParty contained healthcheck_request_body -syn keyword ngxDirectiveThirdParty contained healthcheck_request_headers -syn keyword ngxDirectiveThirdParty contained healthcheck_request_uri -syn keyword ngxDirectiveThirdParty contained healthcheck_response_body -syn keyword ngxDirectiveThirdParty contained healthcheck_response_codes -syn keyword ngxDirectiveThirdParty contained healthcheck_status -syn keyword ngxDirectiveThirdParty contained healthcheck_update - " https://github.com/openresty/encrypted-session-nginx-module syn keyword ngxDirectiveThirdParty contained encrypted_session_expires syn keyword ngxDirectiveThirdParty contained encrypted_session_iv @@ -1004,6 +960,7 @@ syn keyword ngxDirectiveThirdParty contained auth_gss_map_to_local syn keyword ngxDirectiveThirdParty contained auth_gss_realm syn keyword ngxDirectiveThirdParty contained auth_gss_service_ccache syn keyword ngxDirectiveThirdParty contained auth_gss_service_name +syn keyword ngxDirectiveThirdParty contained auth_gss_zone_name " https://github.com/kvspb/nginx-auth-ldap syn keyword ngxDirectiveThirdParty contained auth_ldap @@ -1033,6 +990,7 @@ syn keyword ngxDirectiveThirdParty contained eval_subrequest_in_memory " https://github.com/aperezdc/ngx-fancyindex syn keyword ngxDirectiveThirdParty contained fancyindex +syn keyword ngxDirectiveThirdParty contained fancyindex_case_sensitive syn keyword ngxDirectiveThirdParty contained fancyindex_css_href syn keyword ngxDirectiveThirdParty contained fancyindex_default_sort syn keyword ngxDirectiveThirdParty contained fancyindex_directories_first @@ -1121,6 +1079,7 @@ syn keyword ngxDirectiveThirdParty contained nchan_publisher_upstream_request syn keyword ngxDirectiveThirdParty contained nchan_pubsub syn keyword ngxDirectiveThirdParty contained nchan_pubsub_channel_id syn keyword ngxDirectiveThirdParty contained nchan_pubsub_location +syn keyword ngxDirectiveThirdParty contained nchan_redis_accurate_subscriber_count syn keyword ngxDirectiveThirdParty contained nchan_redis_cluster_check_interval syn keyword ngxDirectiveThirdParty contained nchan_redis_cluster_check_interval_backoff syn keyword ngxDirectiveThirdParty contained nchan_redis_cluster_check_interval_jitter @@ -1138,6 +1097,11 @@ syn keyword ngxDirectiveThirdParty contained nchan_redis_connect_timeout syn keyword ngxDirectiveThirdParty contained nchan_redis_discovered_ip_range_blacklist syn keyword ngxDirectiveThirdParty contained nchan_redis_fakesub_timer_interval syn keyword ngxDirectiveThirdParty contained nchan_redis_idle_channel_cache_timeout +syn keyword ngxDirectiveThirdParty contained nchan_redis_idle_channel_keepalive_backoff +syn keyword ngxDirectiveThirdParty contained nchan_redis_idle_channel_keepalive_jitter +syn keyword ngxDirectiveThirdParty contained nchan_redis_idle_channel_keepalive_max +syn keyword ngxDirectiveThirdParty contained nchan_redis_idle_channel_keepalive_min +syn keyword ngxDirectiveThirdParty contained nchan_redis_idle_channel_keepalive_safety_margin syn keyword ngxDirectiveThirdParty contained nchan_redis_load_scripts_unconditionally syn keyword ngxDirectiveThirdParty contained nchan_redis_namespace syn keyword ngxDirectiveThirdParty contained nchan_redis_node_connect_timeout @@ -1173,6 +1137,9 @@ syn keyword ngxDirectiveThirdParty contained nchan_redis_tls_server_name syn keyword ngxDirectiveThirdParty contained nchan_redis_tls_trusted_certificate syn keyword ngxDirectiveThirdParty contained nchan_redis_tls_trusted_certificate_path syn keyword ngxDirectiveThirdParty contained nchan_redis_tls_verify_certificate +syn keyword ngxDirectiveThirdParty contained nchan_redis_upstream_stats +syn keyword ngxDirectiveThirdParty contained nchan_redis_upstream_stats_disconnected_timeout +syn keyword ngxDirectiveThirdParty contained nchan_redis_upstream_stats_enabled syn keyword ngxDirectiveThirdParty contained nchan_redis_url syn keyword ngxDirectiveThirdParty contained nchan_redis_username syn keyword ngxDirectiveThirdParty contained nchan_redis_wait_after_connecting @@ -1323,6 +1290,7 @@ syn keyword ngxDirectiveThirdParty contained upload_progress_jsonp_parameter syn keyword ngxDirectiveThirdParty contained upload_progress_template " https://github.com/yaoweibin/nginx_upstream_check_module +syn keyword ngxDirectiveThirdParty contained check syn keyword ngxDirectiveThirdParty contained check_fastcgi_param syn keyword ngxDirectiveThirdParty contained check_http_expect_alive syn keyword ngxDirectiveThirdParty contained check_http_send @@ -1335,6 +1303,7 @@ syn keyword ngxDirectiveThirdParty contained fair syn keyword ngxDirectiveThirdParty contained upstream_fair_shm_size " https://github.com/ayty-adrianomartins/nginx-sticky-module-ng +syn keyword ngxDirectiveThirdParty contained sticky_hide_cookie syn keyword ngxDirectiveThirdParty contained sticky_no_fallback " https://github.com/Novetta/nginx-video-thumbextractor-module @@ -1421,6 +1390,8 @@ syn keyword ngxDirectiveThirdParty contained lua_socket_pool_size syn keyword ngxDirectiveThirdParty contained lua_socket_read_timeout syn keyword ngxDirectiveThirdParty contained lua_socket_send_lowat syn keyword ngxDirectiveThirdParty contained lua_socket_send_timeout +syn keyword ngxDirectiveThirdParty contained lua_ssl_certificate +syn keyword ngxDirectiveThirdParty contained lua_ssl_certificate_key syn keyword ngxDirectiveThirdParty contained lua_ssl_ciphers syn keyword ngxDirectiveThirdParty contained lua_ssl_conf_command syn keyword ngxDirectiveThirdParty contained lua_ssl_crl @@ -1834,16 +1805,6 @@ syn keyword ngxDirectiveThirdParty contained slowfs_cache_purge syn keyword ngxDirectiveThirdParty contained slowfs_cache_valid syn keyword ngxDirectiveThirdParty contained slowfs_temp_path -" https://github.com/kawakibi/ngx_small_light -syn keyword ngxDirectiveThirdParty contained small_light -syn keyword ngxDirectiveThirdParty contained small_light_buffer -syn keyword ngxDirectiveThirdParty contained small_light_getparam_mode -syn keyword ngxDirectiveThirdParty contained small_light_imlib2_temp_dir -syn keyword ngxDirectiveThirdParty contained small_light_material_dir -syn keyword ngxDirectiveThirdParty contained small_light_pattern_define -syn keyword ngxDirectiveThirdParty contained small_light_radius_max -syn keyword ngxDirectiveThirdParty contained small_light_sigma_max - " https://github.com/openresty/srcache-nginx-module syn keyword ngxDirectiveThirdParty contained srcache_buffer syn keyword ngxDirectiveThirdParty contained srcache_default_expire @@ -1980,6 +1941,14 @@ syn keyword ngxDirectiveThirdParty contained websockify_pass syn keyword ngxDirectiveThirdParty contained websockify_read_timeout syn keyword ngxDirectiveThirdParty contained websockify_send_timeout +" https://github.com/vozlt/nginx-module-sts +syn keyword ngxDirectiveThirdParty contained stream_server_traffic_status +syn keyword ngxDirectiveThirdParty contained stream_server_traffic_status_average_method +syn keyword ngxDirectiveThirdParty contained stream_server_traffic_status_display +syn keyword ngxDirectiveThirdParty contained stream_server_traffic_status_display_format +syn keyword ngxDirectiveThirdParty contained stream_server_traffic_status_display_jsonp +syn keyword ngxDirectiveThirdParty contained stream_server_traffic_status_zone + " highlight hi def link ngxComment Comment From 2038b46e25b74c16b36ce27f4c8064f2ab2af5a9 Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Wed, 21 Jun 2023 01:29:55 +0300 Subject: [PATCH 020/279] SSL: provided "nginx" appname when loading OpenSSL configs. Following OpenSSL 0.9.8f, OpenSSL tries to load application-specific configuration section first, and then falls back to the "openssl_conf" default section if application-specific section is not found, by using CONF_modules_load_file(CONF_MFLAGS_DEFAULT_SECTION). Therefore this change is not expected to introduce any compatibility issues with existing configurations. It does, however, make it easier to configure specific OpenSSL settings for nginx in system-wide OpenSSL configuration (ticket #2449). Instead of checking OPENSSL_VERSION_NUMBER when using the OPENSSL_init_ssl() interface, the code now tests for OPENSSL_INIT_LOAD_CONFIG to be defined and true, and also explicitly excludes LibreSSL. This ensures that this interface is not used with BoringSSL and LibreSSL, which do not provide additional library initialization settings, notably the OPENSSL_INIT_set_config_appname() call. --- src/event/ngx_event_openssl.c | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c index c38aa27f196..32cdabf0b27 100644 --- a/src/event/ngx_event_openssl.c +++ b/src/event/ngx_event_openssl.c @@ -140,13 +140,31 @@ int ngx_ssl_stapling_index; ngx_int_t ngx_ssl_init(ngx_log_t *log) { -#if OPENSSL_VERSION_NUMBER >= 0x10100003L +#if (OPENSSL_INIT_LOAD_CONFIG && !defined LIBRESSL_VERSION_NUMBER) + + OPENSSL_INIT_SETTINGS *init; + + init = OPENSSL_INIT_new(); + if (init == NULL) { + ngx_ssl_error(NGX_LOG_ALERT, log, 0, "OPENSSL_INIT_new() failed"); + return NGX_ERROR; + } - if (OPENSSL_init_ssl(OPENSSL_INIT_LOAD_CONFIG, NULL) == 0) { +#ifndef OPENSSL_NO_STDIO + if (OPENSSL_INIT_set_config_appname(init, "nginx") == 0) { + ngx_ssl_error(NGX_LOG_ALERT, log, 0, + "OPENSSL_INIT_set_config_appname() failed"); + return NGX_ERROR; + } +#endif + + if (OPENSSL_init_ssl(OPENSSL_INIT_LOAD_CONFIG, init) == 0) { ngx_ssl_error(NGX_LOG_ALERT, log, 0, "OPENSSL_init_ssl() failed"); return NGX_ERROR; } + OPENSSL_INIT_free(init); + /* * OPENSSL_init_ssl() may leave errors in the error queue * while returning success @@ -156,7 +174,7 @@ ngx_ssl_init(ngx_log_t *log) #else - OPENSSL_config(NULL); + OPENSSL_config("nginx"); SSL_library_init(); SSL_load_error_strings(); From bdea5b703ff6f6fcf98ac8dd4e1e9e5c9ad05017 Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Wed, 21 Jun 2023 01:29:53 +0300 Subject: [PATCH 021/279] SSL: avoid using OpenSSL config in build directory (ticket #2404). With this change, the NGX_OPENSSL_NO_CONFIG macro is defined when nginx is asked to build OpenSSL itself. And with this macro automatic loading of OpenSSL configuration (from the build directory) is prevented unless the OPENSSL_CONF environment variable is explicitly set. Note that not loading configuration is broken in OpenSSL 1.1.1 and 1.1.1a (fixed in OpenSSL 1.1.1b, see https://github.com/openssl/openssl/issues/7350). If nginx is used to compile these OpenSSL versions, configuring nginx with NGX_OPENSSL_NO_CONFIG explicitly set to 0 might be used as a workaround. --- auto/lib/openssl/conf | 2 ++ src/event/ngx_event_openssl.c | 21 ++++++++++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/auto/lib/openssl/conf b/auto/lib/openssl/conf index cfa74cf81bc..eda1c0f4ad6 100644 --- a/auto/lib/openssl/conf +++ b/auto/lib/openssl/conf @@ -8,6 +8,8 @@ if [ $OPENSSL != NONE ]; then have=NGX_OPENSSL . auto/have have=NGX_SSL . auto/have + have=NGX_OPENSSL_NO_CONFIG . auto/have + if [ $USE_OPENSSL_QUIC = YES ]; then have=NGX_QUIC . auto/have have=NGX_QUIC_OPENSSL_COMPAT . auto/have diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c index 32cdabf0b27..8468101d1f2 100644 --- a/src/event/ngx_event_openssl.c +++ b/src/event/ngx_event_openssl.c @@ -142,8 +142,19 @@ ngx_ssl_init(ngx_log_t *log) { #if (OPENSSL_INIT_LOAD_CONFIG && !defined LIBRESSL_VERSION_NUMBER) + uint64_t opts; OPENSSL_INIT_SETTINGS *init; + opts = OPENSSL_INIT_LOAD_CONFIG; + +#if (NGX_OPENSSL_NO_CONFIG) + + if (getenv("OPENSSL_CONF") == NULL) { + opts = OPENSSL_INIT_NO_LOAD_CONFIG; + } + +#endif + init = OPENSSL_INIT_new(); if (init == NULL) { ngx_ssl_error(NGX_LOG_ALERT, log, 0, "OPENSSL_INIT_new() failed"); @@ -158,7 +169,7 @@ ngx_ssl_init(ngx_log_t *log) } #endif - if (OPENSSL_init_ssl(OPENSSL_INIT_LOAD_CONFIG, init) == 0) { + if (OPENSSL_init_ssl(opts, init) == 0) { ngx_ssl_error(NGX_LOG_ALERT, log, 0, "OPENSSL_init_ssl() failed"); return NGX_ERROR; } @@ -174,6 +185,14 @@ ngx_ssl_init(ngx_log_t *log) #else +#if (NGX_OPENSSL_NO_CONFIG) + + if (getenv("OPENSSL_CONF") == NULL) { + OPENSSL_no_config(); + } + +#endif + OPENSSL_config("nginx"); SSL_library_init(); From 6e60e21ac09a88a17e08cb4a15ebfcd6634ba10b Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Thu, 27 Jul 2023 16:37:17 +0400 Subject: [PATCH 022/279] QUIC: optimized ACK delay. Previously ACK was not generated if max_ack_delay was not yet expired and the number of unacknowledged ack-eliciting packets was less than two, as allowed by RFC 9000 13.2.1-13.2.2. However this only makes sense to avoid sending ACK-only packets, as explained by the RFC: On the other hand, reducing the frequency of packets that carry only acknowledgments reduces packet transmission and processing cost at both endpoints. Now ACK is delayed only if output frame queue is empty. Otherwise ACK is sent immediately, which significantly improves QUIC performance with certain tests. --- src/event/quic/ngx_event_quic_ack.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/event/quic/ngx_event_quic_ack.c b/src/event/quic/ngx_event_quic_ack.c index f194abfffdb..865be2e6afe 100644 --- a/src/event/quic/ngx_event_quic_ack.c +++ b/src/event/quic/ngx_event_quic_ack.c @@ -1171,7 +1171,8 @@ ngx_quic_generate_ack(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx) delay = ngx_current_msec - ctx->ack_delay_start; qc = ngx_quic_get_connection(c); - if (ctx->send_ack < NGX_QUIC_MAX_ACK_GAP + if (ngx_queue_empty(&ctx->frames) + && ctx->send_ack < NGX_QUIC_MAX_ACK_GAP && delay < qc->tp.max_ack_delay) { if (!qc->push.timer_set && !qc->closing) { From 6f5f17358ee6286e491f287eeda5cf2b3946bab6 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Thu, 10 Aug 2023 20:11:29 +0400 Subject: [PATCH 023/279] QUIC: always add ACK frame to the queue head. Previously it was added to the tail as all other frames. However, if the amount of queued data is large, it could delay the delivery of ACK, which could trigger frames retransmissions and slow down the connection. --- src/event/quic/ngx_event_quic_output.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/event/quic/ngx_event_quic_output.c b/src/event/quic/ngx_event_quic_output.c index 38006a9a505..88b7df6ab10 100644 --- a/src/event/quic/ngx_event_quic_output.c +++ b/src/event/quic/ngx_event_quic_output.c @@ -1174,8 +1174,9 @@ ngx_quic_send_ack(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx) frame->u.ack.delay = ack_delay; frame->u.ack.range_count = ctx->nranges; frame->u.ack.first_range = ctx->first_range; + frame->len = ngx_quic_create_frame(NULL, frame); - ngx_quic_queue_frame(qc, frame); + ngx_queue_insert_head(&ctx->frames, &frame->queue); return NGX_OK; } From 968293d5e721b9c7ff4098ee76a3313b0b7fd2ac Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Thu, 27 Jul 2023 13:35:42 +0400 Subject: [PATCH 024/279] QUIC: fixed congesion control in GSO mode. In non-GSO mode, a datagram is sent if congestion window is not exceeded by the time of send. The window could be exceeded by a small amount after the send. In GSO mode, congestion window was checked in a similar way, but for all concatenated datagrams as a whole. This could result in exceeding congestion window by a lot. Now congestion window is checked for every datagram in GSO mode as well. --- src/event/quic/ngx_event_quic_output.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/event/quic/ngx_event_quic_output.c b/src/event/quic/ngx_event_quic_output.c index 88b7df6ab10..37de44c749e 100644 --- a/src/event/quic/ngx_event_quic_output.c +++ b/src/event/quic/ngx_event_quic_output.c @@ -358,7 +358,7 @@ ngx_quic_create_segments(ngx_connection_t *c) len = ngx_min(segsize, (size_t) (end - p)); - if (len && cg->in_flight < cg->window) { + if (len && cg->in_flight + (p - dst) < cg->window) { n = ngx_quic_output_packet(c, ctx, p, len, len); if (n == NGX_ERROR) { From 57f87d61639d7fc0f5df187056ef03fcec3236a8 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Tue, 1 Aug 2023 11:20:04 +0400 Subject: [PATCH 025/279] QUIC: avoid accessing freed frame. Previously the field pnum of a potentially freed frame was accessed. Now the value is copied to a local variable. The old behavior did not cause any problems since the frame memory is not freed, but is moved to a free queue instead. --- src/event/quic/ngx_event_quic_ack.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/event/quic/ngx_event_quic_ack.c b/src/event/quic/ngx_event_quic_ack.c index 865be2e6afe..e6210653ac8 100644 --- a/src/event/quic/ngx_event_quic_ack.c +++ b/src/event/quic/ngx_event_quic_ack.c @@ -548,6 +548,7 @@ ngx_quic_persistent_congestion(ngx_connection_t *c) void ngx_quic_resend_frames(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx) { + uint64_t pnum; ngx_queue_t *q; ngx_quic_frame_t *f, *start; ngx_quic_stream_t *qs; @@ -556,6 +557,7 @@ ngx_quic_resend_frames(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx) qc = ngx_quic_get_connection(c); q = ngx_queue_head(&ctx->sent); start = ngx_queue_data(q, ngx_quic_frame_t, queue); + pnum = start->pnum; ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic resend packet pnum:%uL", start->pnum); @@ -565,7 +567,7 @@ ngx_quic_resend_frames(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx) do { f = ngx_queue_data(q, ngx_quic_frame_t, queue); - if (f->pnum != start->pnum) { + if (f->pnum != pnum) { break; } From 842a930b88c6103a793d210826813f09d7fa84a7 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Tue, 1 Aug 2023 11:21:59 +0400 Subject: [PATCH 026/279] QUIC: fixed PTO expiration condition. Previously, PTO handler analyzed the first packet in the sent queue for the timeout expiration. However, the last sent packet should be analyzed instead. An example is timeout calculation in ngx_quic_set_lost_timer(). --- src/event/quic/ngx_event_quic_ack.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/event/quic/ngx_event_quic_ack.c b/src/event/quic/ngx_event_quic_ack.c index e6210653ac8..04fc6676018 100644 --- a/src/event/quic/ngx_event_quic_ack.c +++ b/src/event/quic/ngx_event_quic_ack.c @@ -840,7 +840,7 @@ ngx_quic_pto_handler(ngx_event_t *ev) continue; } - q = ngx_queue_head(&ctx->sent); + q = ngx_queue_last(&ctx->sent); f = ngx_queue_data(q, ngx_quic_frame_t, queue); if (f->pnum <= ctx->largest_ack From 4f3707c5c775ac83a0cf005617e1fdffd027b8a4 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Mon, 14 Aug 2023 08:28:30 +0400 Subject: [PATCH 027/279] QUIC: fixed probe-congestion deadlock. When probe timeout expired while congestion window was exhausted, probe PINGs could not be sent. As a result, lost packets could not be declared lost and congestion window could not be freed for new packets. This deadlock continued until connection idle timeout expiration. Now PINGs are sent separately from the frame queue without congestion control, as specified by RFC 9002, Section 7: An endpoint MUST NOT send a packet if it would cause bytes_in_flight (see Appendix B.2) to be larger than the congestion window, unless the packet is sent on a PTO timer expiration (see Section 6.2) or when entering recovery (see Section 7.3.2). --- src/event/quic/ngx_event_quic_ack.c | 62 ++++------------------- src/event/quic/ngx_event_quic_output.c | 4 -- src/event/quic/ngx_event_quic_transport.h | 1 - 3 files changed, 11 insertions(+), 56 deletions(-) diff --git a/src/event/quic/ngx_event_quic_ack.c b/src/event/quic/ngx_event_quic_ack.c index 04fc6676018..182246568fc 100644 --- a/src/event/quic/ngx_event_quic_ack.c +++ b/src/event/quic/ngx_event_quic_ack.c @@ -820,9 +820,9 @@ ngx_quic_pto_handler(ngx_event_t *ev) { ngx_uint_t i; ngx_msec_t now; - ngx_queue_t *q, *next; + ngx_queue_t *q; ngx_connection_t *c; - ngx_quic_frame_t *f; + ngx_quic_frame_t *f, frame; ngx_quic_send_ctx_t *ctx; ngx_quic_connection_t *qc; @@ -859,63 +859,23 @@ ngx_quic_pto_handler(ngx_event_t *ev) "quic pto %s pto_count:%ui", ngx_quic_level_name(ctx->level), qc->pto_count); - for (q = ngx_queue_head(&ctx->frames); - q != ngx_queue_sentinel(&ctx->frames); - /* void */) - { - next = ngx_queue_next(q); - f = ngx_queue_data(q, ngx_quic_frame_t, queue); - - if (f->type == NGX_QUIC_FT_PING) { - ngx_queue_remove(q); - ngx_quic_free_frame(c, f); - } + ngx_memzero(&frame, sizeof(ngx_quic_frame_t)); - q = next; - } + frame.level = ctx->level; + frame.type = NGX_QUIC_FT_PING; - for (q = ngx_queue_head(&ctx->sent); - q != ngx_queue_sentinel(&ctx->sent); - /* void */) + if (ngx_quic_frame_sendto(c, &frame, 0, qc->path) != NGX_OK + || ngx_quic_frame_sendto(c, &frame, 0, qc->path) != NGX_OK) { - next = ngx_queue_next(q); - f = ngx_queue_data(q, ngx_quic_frame_t, queue); - - if (f->type == NGX_QUIC_FT_PING) { - ngx_quic_congestion_lost(c, f); - ngx_queue_remove(q); - ngx_quic_free_frame(c, f); - } - - q = next; - } - - /* enforce 2 udp datagrams */ - - f = ngx_quic_alloc_frame(c); - if (f == NULL) { - break; - } - - f->level = ctx->level; - f->type = NGX_QUIC_FT_PING; - f->flush = 1; - - ngx_quic_queue_frame(qc, f); - - f = ngx_quic_alloc_frame(c); - if (f == NULL) { - break; + ngx_quic_close_connection(c, NGX_ERROR); + return; } - - f->level = ctx->level; - f->type = NGX_QUIC_FT_PING; - - ngx_quic_queue_frame(qc, f); } qc->pto_count++; + ngx_quic_set_lost_timer(c); + ngx_quic_connstate_dbg(c); } diff --git a/src/event/quic/ngx_event_quic_output.c b/src/event/quic/ngx_event_quic_output.c index 37de44c749e..ecd5c2cef46 100644 --- a/src/event/quic/ngx_event_quic_output.c +++ b/src/event/quic/ngx_event_quic_output.c @@ -631,10 +631,6 @@ ngx_quic_output_packet(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx, f->plen = 0; nframes++; - - if (f->flush) { - break; - } } if (nframes == 0) { diff --git a/src/event/quic/ngx_event_quic_transport.h b/src/event/quic/ngx_event_quic_transport.h index 16d9095efca..02e44650bec 100644 --- a/src/event/quic/ngx_event_quic_transport.h +++ b/src/event/quic/ngx_event_quic_transport.h @@ -271,7 +271,6 @@ struct ngx_quic_frame_s { ssize_t len; unsigned need_ack:1; unsigned pkt_need_ack:1; - unsigned flush:1; ngx_chain_t *data; union { From 3990aaaa5557d334f731dcc8f04273a0c45feee3 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Thu, 6 Jul 2023 17:49:01 +0400 Subject: [PATCH 028/279] QUIC: removed path->limited flag. Its value is the opposite of path->validated. --- src/event/quic/ngx_event_quic.c | 1 - src/event/quic/ngx_event_quic_connection.h | 1 - src/event/quic/ngx_event_quic_migration.c | 4 ---- src/event/quic/ngx_event_quic_migration.h | 7 +++---- src/event/quic/ngx_event_quic_output.c | 4 ++-- src/event/quic/ngx_event_quic_socket.c | 1 - 6 files changed, 5 insertions(+), 13 deletions(-) diff --git a/src/event/quic/ngx_event_quic.c b/src/event/quic/ngx_event_quic.c index b559c485d31..25f658fd4ed 100644 --- a/src/event/quic/ngx_event_quic.c +++ b/src/event/quic/ngx_event_quic.c @@ -1013,7 +1013,6 @@ ngx_quic_handle_payload(ngx_connection_t *c, ngx_quic_header_t *pkt) if (!qc->path->validated) { qc->path->validated = 1; - qc->path->limited = 0; ngx_quic_path_dbg(c, "in handshake", qc->path); ngx_post_event(&qc->push, &ngx_posted_events); } diff --git a/src/event/quic/ngx_event_quic_connection.h b/src/event/quic/ngx_event_quic_connection.h index 466f90f93f8..0dd2392e704 100644 --- a/src/event/quic/ngx_event_quic_connection.h +++ b/src/event/quic/ngx_event_quic_connection.h @@ -101,7 +101,6 @@ struct ngx_quic_path_s { u_char text[NGX_SOCKADDR_STRLEN]; unsigned validated:1; unsigned validating:1; - unsigned limited:1; }; diff --git a/src/event/quic/ngx_event_quic_migration.c b/src/event/quic/ngx_event_quic_migration.c index cf0438fbc7d..ca821f8b47b 100644 --- a/src/event/quic/ngx_event_quic_migration.c +++ b/src/event/quic/ngx_event_quic_migration.c @@ -168,7 +168,6 @@ ngx_quic_handle_path_response_frame(ngx_connection_t *c, path->validated = 1; path->validating = 0; - path->limited = 0; ngx_quic_set_path_timer(c); @@ -208,8 +207,6 @@ ngx_quic_new_path(ngx_connection_t *c, path->cid = cid; cid->used = 1; - path->limited = 1; - path->seqnum = qc->path_seqnum++; path->sockaddr = &path->sa.sockaddr; @@ -665,7 +662,6 @@ ngx_quic_path_validation_handler(ngx_event_t *ev) path->validated = 0; path->validating = 0; - path->limited = 1; /* RFC 9000, 9.3.2. On-Path Address Spoofing diff --git a/src/event/quic/ngx_event_quic_migration.h b/src/event/quic/ngx_event_quic_migration.h index 0e1e85454e5..7f95086be3f 100644 --- a/src/event/quic/ngx_event_quic_migration.h +++ b/src/event/quic/ngx_event_quic_migration.h @@ -18,11 +18,10 @@ #define NGX_QUIC_PATH_BACKUP 2 #define ngx_quic_path_dbg(c, msg, path) \ - ngx_log_debug7(NGX_LOG_DEBUG_EVENT, c->log, 0, \ - "quic path seq:%uL %s sent:%O recvd:%O state:%s%s%s", \ + ngx_log_debug6(NGX_LOG_DEBUG_EVENT, c->log, 0, \ + "quic path seq:%uL %s sent:%O recvd:%O state:%s%s", \ path->seqnum, msg, path->sent, path->received, \ - path->limited ? "L" : "", path->validated ? "V": "N", \ - path->validating ? "R": ""); + path->validated ? "V": "N", path->validating ? "R": ""); ngx_int_t ngx_quic_handle_path_challenge_frame(ngx_connection_t *c, ngx_quic_header_t *pkt, ngx_quic_path_challenge_frame_t *f); diff --git a/src/event/quic/ngx_event_quic_output.c b/src/event/quic/ngx_event_quic_output.c index ecd5c2cef46..23235e235e8 100644 --- a/src/event/quic/ngx_event_quic_output.c +++ b/src/event/quic/ngx_event_quic_output.c @@ -281,7 +281,7 @@ ngx_quic_allow_segmentation(ngx_connection_t *c) return 0; } - if (qc->path->limited) { + if (!qc->path->validated) { /* don't even try to be faster on non-validated paths */ return 0; } @@ -1275,7 +1275,7 @@ ngx_quic_path_limit(ngx_connection_t *c, ngx_quic_path_t *path, size_t size) { off_t max; - if (path->limited) { + if (!path->validated) { max = path->received * 3; max = (path->sent >= max) ? 0 : max - path->sent; diff --git a/src/event/quic/ngx_event_quic_socket.c b/src/event/quic/ngx_event_quic_socket.c index 6652523b712..8339de3faef 100644 --- a/src/event/quic/ngx_event_quic_socket.c +++ b/src/event/quic/ngx_event_quic_socket.c @@ -82,7 +82,6 @@ ngx_quic_open_sockets(ngx_connection_t *c, ngx_quic_connection_t *qc, if (pkt->validated) { qc->path->validated = 1; - qc->path->limited = 0; } ngx_quic_path_dbg(c, "set active", qc->path); From 8ab38890736128ec0e167ae28d23d3bbfe2a2223 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Thu, 6 Jul 2023 11:30:47 +0400 Subject: [PATCH 029/279] QUIC: removed explicit packet padding for certain frames. The frames for which the padding is removed are PATH_CHALLENGE and PATH_RESPONSE, which are sent separately by ngx_quic_frame_sendto(). --- src/event/quic/ngx_event_quic_output.c | 30 +------------------------- 1 file changed, 1 insertion(+), 29 deletions(-) diff --git a/src/event/quic/ngx_event_quic_output.c b/src/event/quic/ngx_event_quic_output.c index 23235e235e8..392c7757bce 100644 --- a/src/event/quic/ngx_event_quic_output.c +++ b/src/event/quic/ngx_event_quic_output.c @@ -525,7 +525,7 @@ ngx_quic_output_packet(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx, ssize_t flen; ngx_str_t res; ngx_int_t rc; - ngx_uint_t nframes, expand; + ngx_uint_t nframes; ngx_msec_t now; ngx_queue_t *q; ngx_quic_frame_t *f; @@ -560,7 +560,6 @@ ngx_quic_output_packet(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx, nframes = 0; p = src; len = 0; - expand = 0; for (q = ngx_queue_head(&ctx->frames); q != ngx_queue_sentinel(&ctx->frames); @@ -568,33 +567,6 @@ ngx_quic_output_packet(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx, { f = ngx_queue_data(q, ngx_quic_frame_t, queue); - if (!expand && (f->type == NGX_QUIC_FT_PATH_RESPONSE - || f->type == NGX_QUIC_FT_PATH_CHALLENGE)) - { - /* - * RFC 9000, 8.2.1. Initiating Path Validation - * - * An endpoint MUST expand datagrams that contain a - * PATH_CHALLENGE frame to at least the smallest allowed - * maximum datagram size of 1200 bytes... - * - * (same applies to PATH_RESPONSE frames) - */ - - if (max < 1200) { - /* expanded packet will not fit */ - break; - } - - if (min < 1200) { - min = 1200; - - min_payload = ngx_quic_payload_size(&pkt, min); - } - - expand = 1; - } - if (len >= max_payload) { break; } From 58fc5e2830a0fa70810005ca03fdb9a531719696 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Tue, 8 Aug 2023 10:43:17 +0400 Subject: [PATCH 030/279] QUIC: allowed ngx_quic_frame_sendto() to return NGX_AGAIN. Previously, NGX_AGAIN returned by ngx_quic_send() was treated by ngx_quic_frame_sendto() as error, which triggered errors in its callers. However, a blocked socket is not an error. Now NGX_AGAIN is passed as is to the ngx_quic_frame_sendto() callers, which can safely ignore it. --- src/event/quic/ngx_event_quic_migration.c | 6 +++--- src/event/quic/ngx_event_quic_output.c | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/event/quic/ngx_event_quic_migration.c b/src/event/quic/ngx_event_quic_migration.c index ca821f8b47b..a91b49a1d59 100644 --- a/src/event/quic/ngx_event_quic_migration.c +++ b/src/event/quic/ngx_event_quic_migration.c @@ -46,7 +46,7 @@ ngx_quic_handle_path_challenge_frame(ngx_connection_t *c, * An endpoint MUST expand datagrams that contain a PATH_RESPONSE frame * to at least the smallest allowed maximum datagram size of 1200 bytes. */ - if (ngx_quic_frame_sendto(c, &frame, 1200, pkt->path) != NGX_OK) { + if (ngx_quic_frame_sendto(c, &frame, 1200, pkt->path) == NGX_ERROR) { return NGX_ERROR; } @@ -544,13 +544,13 @@ ngx_quic_send_path_challenge(ngx_connection_t *c, ngx_quic_path_t *path) */ /* same applies to PATH_RESPONSE frames */ - if (ngx_quic_frame_sendto(c, &frame, 1200, path) != NGX_OK) { + if (ngx_quic_frame_sendto(c, &frame, 1200, path) == NGX_ERROR) { return NGX_ERROR; } ngx_memcpy(frame.u.path_challenge.data, path->challenge2, 8); - if (ngx_quic_frame_sendto(c, &frame, 1200, path) != NGX_OK) { + if (ngx_quic_frame_sendto(c, &frame, 1200, path) == NGX_ERROR) { return NGX_ERROR; } diff --git a/src/event/quic/ngx_event_quic_output.c b/src/event/quic/ngx_event_quic_output.c index 392c7757bce..a72504b6a6e 100644 --- a/src/event/quic/ngx_event_quic_output.c +++ b/src/event/quic/ngx_event_quic_output.c @@ -1233,7 +1233,7 @@ ngx_quic_frame_sendto(ngx_connection_t *c, ngx_quic_frame_t *frame, sent = ngx_quic_send(c, res.data, res.len, path->sockaddr, path->socklen); if (sent < 0) { - return NGX_ERROR; + return sent; } path->sent += sent; From eeb8a9f56f727d2543c8ae7feae0de784e329b27 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Mon, 14 Aug 2023 09:21:27 +0400 Subject: [PATCH 031/279] QUIC: path MTU discovery. MTU selection starts by doubling the initial MTU until the first failure. Then binary search is used to find the path MTU. --- src/core/ngx_connection.c | 5 + src/core/ngx_connection.h | 3 +- src/event/quic/ngx_event_quic.c | 9 +- src/event/quic/ngx_event_quic_ack.c | 6 + src/event/quic/ngx_event_quic_connection.h | 16 +- src/event/quic/ngx_event_quic_migration.c | 361 +++++++++++++++++---- src/event/quic/ngx_event_quic_migration.h | 12 +- src/event/quic/ngx_event_quic_output.c | 30 +- src/event/quic/ngx_event_quic_output.h | 2 - src/event/quic/ngx_event_quic_ssl.c | 2 + src/os/unix/ngx_errno.h | 1 + src/os/win32/ngx_errno.h | 1 + 12 files changed, 347 insertions(+), 101 deletions(-) diff --git a/src/core/ngx_connection.c b/src/core/ngx_connection.c index 10f4d9b9136..75809d9ad96 100644 --- a/src/core/ngx_connection.c +++ b/src/core/ngx_connection.c @@ -1583,6 +1583,10 @@ ngx_connection_error(ngx_connection_t *c, ngx_err_t err, char *text) } #endif + if (err == NGX_EMSGSIZE && c->log_error == NGX_ERROR_IGNORE_EMSGSIZE) { + return 0; + } + if (err == 0 || err == NGX_ECONNRESET #if (NGX_WIN32) @@ -1600,6 +1604,7 @@ ngx_connection_error(ngx_connection_t *c, ngx_err_t err, char *text) { switch (c->log_error) { + case NGX_ERROR_IGNORE_EMSGSIZE: case NGX_ERROR_IGNORE_EINVAL: case NGX_ERROR_IGNORE_ECONNRESET: case NGX_ERROR_INFO: diff --git a/src/core/ngx_connection.h b/src/core/ngx_connection.h index c90f0ea50aa..84dd80442fb 100644 --- a/src/core/ngx_connection.h +++ b/src/core/ngx_connection.h @@ -97,7 +97,8 @@ typedef enum { NGX_ERROR_ERR, NGX_ERROR_INFO, NGX_ERROR_IGNORE_ECONNRESET, - NGX_ERROR_IGNORE_EINVAL + NGX_ERROR_IGNORE_EINVAL, + NGX_ERROR_IGNORE_EMSGSIZE } ngx_connection_log_error_e; diff --git a/src/event/quic/ngx_event_quic.c b/src/event/quic/ngx_event_quic.c index 25f658fd4ed..fb211cc9d1a 100644 --- a/src/event/quic/ngx_event_quic.c +++ b/src/event/quic/ngx_event_quic.c @@ -149,11 +149,6 @@ ngx_quic_apply_transport_params(ngx_connection_t *c, ngx_quic_tp_t *ctp) ngx_log_error(NGX_LOG_INFO, c->log, 0, "quic maximum packet size is invalid"); return NGX_ERROR; - - } else if (ctp->max_udp_payload_size > ngx_quic_max_udp_payload(c)) { - ctp->max_udp_payload_size = ngx_quic_max_udp_payload(c); - ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, - "quic client maximum packet size truncated"); } if (ctp->active_connection_id_limit < 2) { @@ -286,7 +281,7 @@ ngx_quic_new_connection(ngx_connection_t *c, ngx_quic_conf_t *conf, qc->path_validation.log = c->log; qc->path_validation.data = c; - qc->path_validation.handler = ngx_quic_path_validation_handler; + qc->path_validation.handler = ngx_quic_path_handler; qc->conf = conf; @@ -297,7 +292,7 @@ ngx_quic_new_connection(ngx_connection_t *c, ngx_quic_conf_t *conf, ctp = &qc->ctp; /* defaults to be used before actual client parameters are received */ - ctp->max_udp_payload_size = ngx_quic_max_udp_payload(c); + ctp->max_udp_payload_size = NGX_QUIC_MAX_UDP_PAYLOAD_SIZE; ctp->ack_delay_exponent = NGX_QUIC_DEFAULT_ACK_DELAY_EXPONENT; ctp->max_ack_delay = NGX_QUIC_DEFAULT_MAX_ACK_DELAY; ctp->active_connection_id_limit = 2; diff --git a/src/event/quic/ngx_event_quic_ack.c b/src/event/quic/ngx_event_quic_ack.c index 182246568fc..23c9af34829 100644 --- a/src/event/quic/ngx_event_quic_ack.c +++ b/src/event/quic/ngx_event_quic_ack.c @@ -229,6 +229,12 @@ ngx_quic_handle_ack_frame_range(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx, qc = ngx_quic_get_connection(c); + if (ctx->level == ssl_encryption_application) { + if (ngx_quic_handle_path_mtu(c, qc->path, min, max) != NGX_OK) { + return NGX_ERROR; + } + } + st->max_pn = NGX_TIMER_INFINITE; found = 0; diff --git a/src/event/quic/ngx_event_quic_connection.h b/src/event/quic/ngx_event_quic_connection.h index 0dd2392e704..5836f94a978 100644 --- a/src/event/quic/ngx_event_quic_connection.h +++ b/src/event/quic/ngx_event_quic_connection.h @@ -66,6 +66,14 @@ typedef struct ngx_quic_keys_s ngx_quic_keys_t; #define ngx_quic_get_socket(c) ((ngx_quic_socket_t *)((c)->udp)) +typedef enum { + NGX_QUIC_PATH_IDLE = 0, + NGX_QUIC_PATH_VALIDATING, + NGX_QUIC_PATH_WAITING, + NGX_QUIC_PATH_MTUD +} ngx_quic_path_state_e; + + struct ngx_quic_client_id_s { ngx_queue_t queue; uint64_t seqnum; @@ -89,18 +97,22 @@ struct ngx_quic_path_s { ngx_sockaddr_t sa; socklen_t socklen; ngx_quic_client_id_t *cid; + ngx_quic_path_state_e state; ngx_msec_t expires; ngx_uint_t tries; ngx_uint_t tag; + size_t mtu; + size_t mtud; + size_t max_mtu; off_t sent; off_t received; u_char challenge1[8]; u_char challenge2[8]; uint64_t seqnum; + uint64_t mtu_pnum[NGX_QUIC_PATH_RETRIES]; ngx_str_t addr_text; u_char text[NGX_SOCKADDR_STRLEN]; - unsigned validated:1; - unsigned validating:1; + ngx_uint_t validated; /* unsigned validated:1; */ }; diff --git a/src/event/quic/ngx_event_quic_migration.c b/src/event/quic/ngx_event_quic_migration.c index a91b49a1d59..05b9a2863e1 100644 --- a/src/event/quic/ngx_event_quic_migration.c +++ b/src/event/quic/ngx_event_quic_migration.c @@ -10,6 +10,10 @@ #include +#define NGX_QUIC_PATH_MTU_DELAY 100 +#define NGX_QUIC_PATH_MTU_PRECISION 16 + + static void ngx_quic_set_connection_path(ngx_connection_t *c, ngx_quic_path_t *path); static ngx_int_t ngx_quic_validate_path(ngx_connection_t *c, @@ -17,7 +21,15 @@ static ngx_int_t ngx_quic_validate_path(ngx_connection_t *c, static ngx_int_t ngx_quic_send_path_challenge(ngx_connection_t *c, ngx_quic_path_t *path); static void ngx_quic_set_path_timer(ngx_connection_t *c); +static ngx_int_t ngx_quic_expire_path_validation(ngx_connection_t *c, + ngx_quic_path_t *path); +static ngx_int_t ngx_quic_expire_path_mtu_delay(ngx_connection_t *c, + ngx_quic_path_t *path); +static ngx_int_t ngx_quic_expire_path_mtu_discovery(ngx_connection_t *c, + ngx_quic_path_t *path); static ngx_quic_path_t *ngx_quic_get_path(ngx_connection_t *c, ngx_uint_t tag); +static ngx_int_t ngx_quic_send_path_mtu_probe(ngx_connection_t *c, + ngx_quic_path_t *path); ngx_int_t @@ -97,7 +109,7 @@ ngx_quic_handle_path_response_frame(ngx_connection_t *c, { path = ngx_queue_data(q, ngx_quic_path_t, queue); - if (!path->validating) { + if (path->state != NGX_QUIC_PATH_VALIDATING) { continue; } @@ -136,6 +148,9 @@ ngx_quic_handle_path_response_frame(ngx_connection_t *c, { /* address did not change */ rst = 0; + + path->mtu = prev->mtu; + path->max_mtu = prev->max_mtu; } } @@ -167,9 +182,8 @@ ngx_quic_handle_path_response_frame(ngx_connection_t *c, ngx_quic_path_dbg(c, "is validated", path); path->validated = 1; - path->validating = 0; - ngx_quic_set_path_timer(c); + ngx_quic_discover_path_mtu(c, path); return NGX_OK; } @@ -217,6 +231,8 @@ ngx_quic_new_path(ngx_connection_t *c, path->addr_text.len = ngx_sock_ntop(sockaddr, socklen, path->text, NGX_SOCKADDR_STRLEN, 1); + path->mtu = NGX_QUIC_MIN_INITIAL_SIZE; + ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic path seq:%uL created addr:%V", path->seqnum, &path->addr_text); @@ -464,7 +480,7 @@ ngx_quic_handle_migration(ngx_connection_t *c, ngx_quic_header_t *pkt) ngx_quic_set_connection_path(c, next); - if (!next->validated && !next->validating) { + if (!next->validated && next->state != NGX_QUIC_PATH_VALIDATING) { if (ngx_quic_validate_path(c, next) != NGX_OK) { return NGX_ERROR; } @@ -492,7 +508,6 @@ ngx_quic_validate_path(ngx_connection_t *c, ngx_quic_path_t *path) ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic initiated validation of path seq:%uL", path->seqnum); - path->validating = 1; path->tries = 0; if (RAND_bytes(path->challenge1, 8) != 1) { @@ -511,6 +526,7 @@ ngx_quic_validate_path(ngx_connection_t *c, ngx_quic_path_t *path) pto = ngx_max(ngx_quic_pto(c, ctx), 1000); path->expires = ngx_current_msec + pto; + path->state = NGX_QUIC_PATH_VALIDATING; ngx_quic_set_path_timer(c); @@ -558,6 +574,42 @@ ngx_quic_send_path_challenge(ngx_connection_t *c, ngx_quic_path_t *path) } +void +ngx_quic_discover_path_mtu(ngx_connection_t *c, ngx_quic_path_t *path) +{ + ngx_quic_connection_t *qc; + + qc = ngx_quic_get_connection(c); + + if (path->max_mtu) { + if (path->max_mtu - path->mtu <= NGX_QUIC_PATH_MTU_PRECISION) { + path->state = NGX_QUIC_PATH_IDLE; + ngx_quic_set_path_timer(c); + return; + } + + path->mtud = (path->mtu + path->max_mtu) / 2; + + } else { + path->mtud = path->mtu * 2; + + if (path->mtud >= qc->ctp.max_udp_payload_size) { + path->mtud = qc->ctp.max_udp_payload_size; + path->max_mtu = qc->ctp.max_udp_payload_size; + } + } + + path->state = NGX_QUIC_PATH_WAITING; + path->expires = ngx_current_msec + NGX_QUIC_PATH_MTU_DELAY; + + ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic path seq:%uL schedule mtu:%uz", + path->seqnum, path->mtud); + + ngx_quic_set_path_timer(c); +} + + static void ngx_quic_set_path_timer(ngx_connection_t *c) { @@ -578,7 +630,7 @@ ngx_quic_set_path_timer(ngx_connection_t *c) { path = ngx_queue_data(q, ngx_quic_path_t, queue); - if (!path->validating) { + if (path->state == NGX_QUIC_PATH_IDLE) { continue; } @@ -600,22 +652,18 @@ ngx_quic_set_path_timer(ngx_connection_t *c) void -ngx_quic_path_validation_handler(ngx_event_t *ev) +ngx_quic_path_handler(ngx_event_t *ev) { ngx_msec_t now; ngx_queue_t *q; - ngx_msec_int_t left, next, pto; - ngx_quic_path_t *path, *bkp; + ngx_msec_int_t left; + ngx_quic_path_t *path; ngx_connection_t *c; - ngx_quic_send_ctx_t *ctx; ngx_quic_connection_t *qc; c = ev->data; qc = ngx_quic_get_connection(c); - ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_application); - - next = -1; now = ngx_current_msec; q = ngx_queue_head(&qc->paths); @@ -625,83 +673,280 @@ ngx_quic_path_validation_handler(ngx_event_t *ev) path = ngx_queue_data(q, ngx_quic_path_t, queue); q = ngx_queue_next(q); - if (!path->validating) { + if (path->state == NGX_QUIC_PATH_IDLE) { continue; } left = path->expires - now; if (left > 0) { + continue; + } - if (next == -1 || left < next) { - next = left; + switch (path->state) { + case NGX_QUIC_PATH_VALIDATING: + if (ngx_quic_expire_path_validation(c, path) != NGX_OK) { + goto failed; } - continue; - } + break; - if (++path->tries < NGX_QUIC_PATH_RETRIES) { - pto = ngx_max(ngx_quic_pto(c, ctx), 1000) << path->tries; + case NGX_QUIC_PATH_WAITING: + if (ngx_quic_expire_path_mtu_delay(c, path) != NGX_OK) { + goto failed; + } - path->expires = ngx_current_msec + pto; + break; - if (next == -1 || pto < next) { - next = pto; + case NGX_QUIC_PATH_MTUD: + if (ngx_quic_expire_path_mtu_discovery(c, path) != NGX_OK) { + goto failed; } - /* retransmit */ - (void) ngx_quic_send_path_challenge(c, path); + break; - continue; + default: + break; } + } - ngx_log_debug1(NGX_LOG_DEBUG_EVENT, ev->log, 0, - "quic path seq:%uL validation failed", path->seqnum); + ngx_quic_set_path_timer(c); - /* found expired path */ + return; - path->validated = 0; - path->validating = 0; +failed: + ngx_quic_close_connection(c, NGX_ERROR); +} - /* RFC 9000, 9.3.2. On-Path Address Spoofing - * - * To protect the connection from failing due to such a spurious - * migration, an endpoint MUST revert to using the last validated - * peer address when validation of a new peer address fails. - */ - if (qc->path == path) { - /* active path validation failed */ +static ngx_int_t +ngx_quic_expire_path_validation(ngx_connection_t *c, ngx_quic_path_t *path) +{ + ngx_msec_int_t pto; + ngx_quic_path_t *bkp; + ngx_quic_send_ctx_t *ctx; + ngx_quic_connection_t *qc; - bkp = ngx_quic_get_path(c, NGX_QUIC_PATH_BACKUP); + qc = ngx_quic_get_connection(c); + ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_application); - if (bkp == NULL) { - qc->error = NGX_QUIC_ERR_NO_VIABLE_PATH; - qc->error_reason = "no viable path"; - ngx_quic_close_connection(c, NGX_ERROR); - return; - } + if (++path->tries < NGX_QUIC_PATH_RETRIES) { + pto = ngx_max(ngx_quic_pto(c, ctx), 1000) << path->tries; + path->expires = ngx_current_msec + pto; + + (void) ngx_quic_send_path_challenge(c, path); + + return NGX_OK; + } - qc->path = bkp; - qc->path->tag = NGX_QUIC_PATH_ACTIVE; + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic path seq:%uL validation failed", path->seqnum); - ngx_quic_set_connection_path(c, qc->path); + /* found expired path */ + + path->validated = 0; + + + /* RFC 9000, 9.3.2. On-Path Address Spoofing + * + * To protect the connection from failing due to such a spurious + * migration, an endpoint MUST revert to using the last validated + * peer address when validation of a new peer address fails. + */ - ngx_log_error(NGX_LOG_INFO, c->log, 0, - "quic path seq:%uL addr:%V is restored from backup", - qc->path->seqnum, &qc->path->addr_text); + if (qc->path == path) { + /* active path validation failed */ - ngx_quic_path_dbg(c, "is active", qc->path); + bkp = ngx_quic_get_path(c, NGX_QUIC_PATH_BACKUP); + + if (bkp == NULL) { + qc->error = NGX_QUIC_ERR_NO_VIABLE_PATH; + qc->error_reason = "no viable path"; + return NGX_ERROR; } - if (ngx_quic_free_path(c, path) != NGX_OK) { - ngx_quic_close_connection(c, NGX_ERROR); - return; + qc->path = bkp; + qc->path->tag = NGX_QUIC_PATH_ACTIVE; + + ngx_quic_set_connection_path(c, qc->path); + + ngx_log_error(NGX_LOG_INFO, c->log, 0, + "quic path seq:%uL addr:%V is restored from backup", + qc->path->seqnum, &qc->path->addr_text); + + ngx_quic_path_dbg(c, "is active", qc->path); + } + + return ngx_quic_free_path(c, path); +} + + +static ngx_int_t +ngx_quic_expire_path_mtu_delay(ngx_connection_t *c, ngx_quic_path_t *path) +{ + ngx_int_t rc; + ngx_uint_t i; + ngx_msec_t pto; + ngx_quic_send_ctx_t *ctx; + ngx_quic_connection_t *qc; + + qc = ngx_quic_get_connection(c); + ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_application); + + path->tries = 0; + + for ( ;; ) { + + for (i = 0; i < NGX_QUIC_PATH_RETRIES; i++) { + path->mtu_pnum[i] = NGX_QUIC_UNSET_PN; + } + + rc = ngx_quic_send_path_mtu_probe(c, path); + + if (rc == NGX_ERROR) { + return NGX_ERROR; + } + + if (rc == NGX_OK) { + pto = ngx_quic_pto(c, ctx); + path->expires = ngx_current_msec + pto; + path->state = NGX_QUIC_PATH_MTUD; + return NGX_OK; + } + + /* rc == NGX_DECLINED */ + + path->max_mtu = path->mtud; + + if (path->max_mtu - path->mtu <= NGX_QUIC_PATH_MTU_PRECISION) { + path->state = NGX_QUIC_PATH_IDLE; + return NGX_OK; } + + path->mtud = (path->mtu + path->max_mtu) / 2; } +} - if (next != -1) { - ngx_add_timer(&qc->path_validation, next); + +static ngx_int_t +ngx_quic_expire_path_mtu_discovery(ngx_connection_t *c, ngx_quic_path_t *path) +{ + ngx_int_t rc; + ngx_msec_int_t pto; + ngx_quic_send_ctx_t *ctx; + ngx_quic_connection_t *qc; + + qc = ngx_quic_get_connection(c); + ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_application); + + if (++path->tries < NGX_QUIC_PATH_RETRIES) { + rc = ngx_quic_send_path_mtu_probe(c, path); + + if (rc == NGX_ERROR) { + return NGX_ERROR; + } + + if (rc == NGX_OK) { + pto = ngx_quic_pto(c, ctx) << path->tries; + path->expires = ngx_current_msec + pto; + return NGX_OK; + } + + /* rc == NGX_DECLINED */ } + + ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic path seq:%uL expired mtu:%uz", + path->seqnum, path->mtud); + + path->max_mtu = path->mtud; + + ngx_quic_discover_path_mtu(c, path); + + return NGX_OK; +} + + +static ngx_int_t +ngx_quic_send_path_mtu_probe(ngx_connection_t *c, ngx_quic_path_t *path) +{ + ngx_int_t rc; + ngx_uint_t log_error; + ngx_quic_frame_t frame; + ngx_quic_send_ctx_t *ctx; + ngx_quic_connection_t *qc; + + ngx_memzero(&frame, sizeof(ngx_quic_frame_t)); + + frame.level = ssl_encryption_application; + frame.type = NGX_QUIC_FT_PING; + + qc = ngx_quic_get_connection(c); + ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_application); + path->mtu_pnum[path->tries] = ctx->pnum; + + ngx_log_debug4(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic path seq:%uL send probe " + "mtu:%uz pnum:%uL tries:%ui", + path->seqnum, path->mtud, ctx->pnum, path->tries); + + log_error = c->log_error; + c->log_error = NGX_ERROR_IGNORE_EMSGSIZE; + + rc = ngx_quic_frame_sendto(c, &frame, path->mtud, path); + c->log_error = log_error; + + if (rc == NGX_ERROR) { + if (c->write->error) { + c->write->error = 0; + + ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic path seq:%uL rejected mtu:%uz", + path->seqnum, path->mtud); + + return NGX_DECLINED; + } + + return NGX_ERROR; + } + + return NGX_OK; +} + + +ngx_int_t +ngx_quic_handle_path_mtu(ngx_connection_t *c, ngx_quic_path_t *path, + uint64_t min, uint64_t max) +{ + uint64_t pnum; + ngx_uint_t i; + + if (path->state != NGX_QUIC_PATH_MTUD) { + return NGX_OK; + } + + for (i = 0; i < NGX_QUIC_PATH_RETRIES; i++) { + pnum = path->mtu_pnum[i]; + + if (pnum == NGX_QUIC_UNSET_PN) { + break; + } + + if (pnum < min || pnum > max) { + continue; + } + + path->mtu = path->mtud; + + ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic path seq:%uL ack mtu:%uz", + path->seqnum, path->mtu); + + ngx_quic_discover_path_mtu(c, path); + + break; + } + + return NGX_OK; } diff --git a/src/event/quic/ngx_event_quic_migration.h b/src/event/quic/ngx_event_quic_migration.h index 7f95086be3f..9587882f280 100644 --- a/src/event/quic/ngx_event_quic_migration.h +++ b/src/event/quic/ngx_event_quic_migration.h @@ -18,10 +18,10 @@ #define NGX_QUIC_PATH_BACKUP 2 #define ngx_quic_path_dbg(c, msg, path) \ - ngx_log_debug6(NGX_LOG_DEBUG_EVENT, c->log, 0, \ - "quic path seq:%uL %s sent:%O recvd:%O state:%s%s", \ + ngx_log_debug7(NGX_LOG_DEBUG_EVENT, c->log, 0, \ + "quic path seq:%uL %s tx:%O rx:%O valid:%ui st:%d mtu:%uz",\ path->seqnum, msg, path->sent, path->received, \ - path->validated ? "V": "N", path->validating ? "R": ""); + path->validated, path->state, path->mtu); ngx_int_t ngx_quic_handle_path_challenge_frame(ngx_connection_t *c, ngx_quic_header_t *pkt, ngx_quic_path_challenge_frame_t *f); @@ -36,6 +36,10 @@ ngx_int_t ngx_quic_set_path(ngx_connection_t *c, ngx_quic_header_t *pkt); ngx_int_t ngx_quic_handle_migration(ngx_connection_t *c, ngx_quic_header_t *pkt); -void ngx_quic_path_validation_handler(ngx_event_t *ev); +void ngx_quic_path_handler(ngx_event_t *ev); + +void ngx_quic_discover_path_mtu(ngx_connection_t *c, ngx_quic_path_t *path); +ngx_int_t ngx_quic_handle_path_mtu(ngx_connection_t *c, + ngx_quic_path_t *path, uint64_t min, uint64_t max); #endif /* _NGX_EVENT_QUIC_MIGRATION_H_INCLUDED_ */ diff --git a/src/event/quic/ngx_event_quic_output.c b/src/event/quic/ngx_event_quic_output.c index a72504b6a6e..587671bc6a9 100644 --- a/src/event/quic/ngx_event_quic_output.c +++ b/src/event/quic/ngx_event_quic_output.c @@ -10,9 +10,6 @@ #include -#define NGX_QUIC_MAX_UDP_PAYLOAD_OUT 1252 -#define NGX_QUIC_MAX_UDP_PAYLOAD_OUT6 1232 - #define NGX_QUIC_MAX_UDP_SEGMENT_BUF 65487 /* 65K - IPv6 header */ #define NGX_QUIC_MAX_SEGMENTS 64 /* UDP_MAX_SEGMENTS */ @@ -61,21 +58,6 @@ static size_t ngx_quic_path_limit(ngx_connection_t *c, ngx_quic_path_t *path, size_t size); -size_t -ngx_quic_max_udp_payload(ngx_connection_t *c) -{ - /* TODO: path MTU discovery */ - -#if (NGX_HAVE_INET6) - if (c->sockaddr->sa_family == AF_INET6) { - return NGX_QUIC_MAX_UDP_PAYLOAD_OUT6; - } -#endif - - return NGX_QUIC_MAX_UDP_PAYLOAD_OUT; -} - - ngx_int_t ngx_quic_output(ngx_connection_t *c) { @@ -142,10 +124,7 @@ ngx_quic_create_datagrams(ngx_connection_t *c) p = dst; - len = ngx_min(qc->ctp.max_udp_payload_size, - NGX_QUIC_MAX_UDP_PAYLOAD_SIZE); - - len = ngx_quic_path_limit(c, path, len); + len = ngx_quic_path_limit(c, path, path->mtu); pad = ngx_quic_get_padding_level(c); @@ -299,9 +278,7 @@ ngx_quic_allow_segmentation(ngx_connection_t *c) ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_application); bytes = 0; - - len = ngx_min(qc->ctp.max_udp_payload_size, - NGX_QUIC_MAX_UDP_SEGMENT_BUF); + len = ngx_min(qc->path->mtu, NGX_QUIC_MAX_UDP_SEGMENT_BUF); for (q = ngx_queue_head(&ctx->frames); q != ngx_queue_sentinel(&ctx->frames); @@ -345,8 +322,7 @@ ngx_quic_create_segments(ngx_connection_t *c) return NGX_ERROR; } - segsize = ngx_min(qc->ctp.max_udp_payload_size, - NGX_QUIC_MAX_UDP_SEGMENT_BUF); + segsize = ngx_min(path->mtu, NGX_QUIC_MAX_UDP_SEGMENT_BUF); p = dst; end = dst + sizeof(dst); diff --git a/src/event/quic/ngx_event_quic_output.h b/src/event/quic/ngx_event_quic_output.h index c19f14bf1df..19f8990f46b 100644 --- a/src/event/quic/ngx_event_quic_output.h +++ b/src/event/quic/ngx_event_quic_output.h @@ -12,8 +12,6 @@ #include -size_t ngx_quic_max_udp_payload(ngx_connection_t *c); - ngx_int_t ngx_quic_output(ngx_connection_t *c); ngx_int_t ngx_quic_negotiate_version(ngx_connection_t *c, diff --git a/src/event/quic/ngx_event_quic_ssl.c b/src/event/quic/ngx_event_quic_ssl.c index 6b0bae1ed19..e0862e29688 100644 --- a/src/event/quic/ngx_event_quic_ssl.c +++ b/src/event/quic/ngx_event_quic_ssl.c @@ -494,6 +494,8 @@ ngx_quic_crypto_input(ngx_connection_t *c, ngx_chain_t *data) */ ngx_quic_discard_ctx(c, ssl_encryption_handshake); + ngx_quic_discover_path_mtu(c, qc->path); + /* start accepting clients on negotiated number of server ids */ if (ngx_quic_create_sockets(c) != NGX_OK) { return NGX_ERROR; diff --git a/src/os/unix/ngx_errno.h b/src/os/unix/ngx_errno.h index 7d6ca764d68..07fa8ced116 100644 --- a/src/os/unix/ngx_errno.h +++ b/src/os/unix/ngx_errno.h @@ -54,6 +54,7 @@ typedef int ngx_err_t; #define NGX_ENOMOREFILES 0 #define NGX_ELOOP ELOOP #define NGX_EBADF EBADF +#define NGX_EMSGSIZE EMSGSIZE #if (NGX_HAVE_OPENAT) #define NGX_EMLINK EMLINK diff --git a/src/os/win32/ngx_errno.h b/src/os/win32/ngx_errno.h index 255a39d5348..1e73a832b02 100644 --- a/src/os/win32/ngx_errno.h +++ b/src/os/win32/ngx_errno.h @@ -57,6 +57,7 @@ typedef DWORD ngx_err_t; #define NGX_EILSEQ ERROR_NO_UNICODE_TRANSLATION #define NGX_ELOOP 0 #define NGX_EBADF WSAEBADF +#define NGX_EMSGSIZE WSAEMSGSIZE #define NGX_EALREADY WSAEALREADY #define NGX_EINVAL WSAEINVAL From e58d3cdd4ebbb81570548d2c19858a7f5d9f8859 Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Tue, 15 Aug 2023 18:10:50 +0300 Subject: [PATCH 032/279] Updated OpenSSL used for win32 builds. --- misc/GNUmakefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misc/GNUmakefile b/misc/GNUmakefile index bd7ce5d763b..0853b40c2cb 100644 --- a/misc/GNUmakefile +++ b/misc/GNUmakefile @@ -6,7 +6,7 @@ TEMP = tmp CC = cl OBJS = objs.msvc8 -OPENSSL = openssl-1.1.1t +OPENSSL = openssl-3.0.10 ZLIB = zlib-1.2.13 PCRE = pcre2-10.39 @@ -105,7 +105,7 @@ zip: export $(MAKE) -f docs/GNUmakefile changes mv $(TEMP)/$(NGINX)/CHANGES* $(TEMP)/$(NGINX)/docs/ - cp -p $(OBJS)/lib/$(OPENSSL)/LICENSE \ + cp -p $(OBJS)/lib/$(OPENSSL)/LICENSE.txt \ $(TEMP)/$(NGINX)/docs/OpenSSL.LICENSE cp -p $(OBJS)/lib/$(PCRE)/LICENCE \ From 349c63ec6118027f1bff2424b1195c7f9a357c73 Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Tue, 15 Aug 2023 20:03:04 +0300 Subject: [PATCH 033/279] nginx-1.25.2-RELEASE --- docs/xml/nginx/changes.xml | 65 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/docs/xml/nginx/changes.xml b/docs/xml/nginx/changes.xml index 1d199361fed..4e9919d2fb9 100644 --- a/docs/xml/nginx/changes.xml +++ b/docs/xml/nginx/changes.xml @@ -5,6 +5,71 @@ + + + + +path MTU discovery при использовании HTTP/3. + + +path MTU discovery when using HTTP/3. + + + + + +поддержка шифра TLS_AES_128_CCM_SHA256 при использовании HTTP/3. + + +TLS_AES_128_CCM_SHA256 cipher suite support when using HTTP/3. + + + + + +теперь при загрузке конфигурации OpenSSL +nginx использует appname "nginx". + + +now nginx uses appname "nginx" +when loading OpenSSL configuration. + + + + + +теперь nginx не пытается загружать конфигурацию OpenSSL, +если для сборки OpenSSL использовался параметр --with-openssl +и переменная окружения OPENSSL_CONF не установлена. + + +now nginx does not try to load OpenSSL configuration +if the --with-openssl option was used to built OpenSSL +and the OPENSSL_CONF environment variable is not set. + + + + + +в переменной $body_bytes_sent при использовании HTTP/3. + + +in the $body_bytes_sent variable when using HTTP/3. + + + + + +в HTTP/3. + + +in HTTP/3. + + + + + + From e5fc65976a479d3ae46f4ffb6334c2851d654234 Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Tue, 15 Aug 2023 20:03:04 +0300 Subject: [PATCH 034/279] release-1.25.2 tag --- .hgtags | 1 + 1 file changed, 1 insertion(+) diff --git a/.hgtags b/.hgtags index cfa3b321e51..4275d556291 100644 --- a/.hgtags +++ b/.hgtags @@ -474,3 +474,4 @@ ff3afd1ce6a6b65057741df442adfaa71a0e2588 release-1.23.3 ac779115ed6ee4f3039e9aea414a54e560450ee2 release-1.23.4 12dcf92b0c2c68552398f19644ce3104459807d7 release-1.25.0 f8134640e8615448205785cf00b0bc810489b495 release-1.25.1 +1d839f05409d1a50d0f15a2bf36547001f99ae40 release-1.25.2 From f42519ff54c0ffecc2751f7957a5c3c9f1c53dfb Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Fri, 25 Aug 2023 16:39:14 +0400 Subject: [PATCH 035/279] Version bump. --- src/core/nginx.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/nginx.h b/src/core/nginx.h index dba95c4a19a..088aa1ecd32 100644 --- a/src/core/nginx.h +++ b/src/core/nginx.h @@ -9,8 +9,8 @@ #define _NGINX_H_INCLUDED_ -#define nginx_version 1025002 -#define NGINX_VERSION "1.25.2" +#define nginx_version 1025003 +#define NGINX_VERSION "1.25.3" #define NGINX_VER "nginx/" NGINX_VERSION #ifdef NGX_BUILD From 24f3cb795e2d822be39387d828ac2ddc28f775fb Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Fri, 25 Aug 2023 13:51:38 +0400 Subject: [PATCH 036/279] QUIC: posted generating TLS Key Update next keys. Since at least f9fbeb4ee0de and certainly after 924882f42dea, which TLS Key Update support predates, queued data output is deferred to a posted push handler. To address timing signals after these changes, generating next keys is now posted to run after the push handler. --- src/event/quic/ngx_event_quic.c | 12 ++++++++- src/event/quic/ngx_event_quic_connection.h | 2 ++ src/event/quic/ngx_event_quic_protection.c | 31 +++++++++++++++------- src/event/quic/ngx_event_quic_protection.h | 2 +- src/event/quic/ngx_event_quic_ssl.c | 4 +-- 5 files changed, 37 insertions(+), 14 deletions(-) diff --git a/src/event/quic/ngx_event_quic.c b/src/event/quic/ngx_event_quic.c index fb211cc9d1a..2d7cecbc00d 100644 --- a/src/event/quic/ngx_event_quic.c +++ b/src/event/quic/ngx_event_quic.c @@ -283,6 +283,10 @@ ngx_quic_new_connection(ngx_connection_t *c, ngx_quic_conf_t *conf, qc->path_validation.data = c; qc->path_validation.handler = ngx_quic_path_handler; + qc->key_update.log = c->log; + qc->key_update.data = c; + qc->key_update.handler = ngx_quic_keys_update; + qc->conf = conf; if (ngx_quic_init_transport_params(&qc->tp, conf) != NGX_OK) { @@ -562,6 +566,10 @@ ngx_quic_close_connection(ngx_connection_t *c, ngx_int_t rc) ngx_delete_posted_event(&qc->push); } + if (qc->key_update.posted) { + ngx_delete_posted_event(&qc->key_update); + } + if (qc->close.timer_set) { return; } @@ -1055,7 +1063,9 @@ ngx_quic_handle_payload(ngx_connection_t *c, ngx_quic_header_t *pkt) return rc; } - return ngx_quic_keys_update(c, qc->keys); + ngx_post_event(&qc->key_update, &ngx_posted_events); + + return NGX_OK; } diff --git a/src/event/quic/ngx_event_quic_connection.h b/src/event/quic/ngx_event_quic_connection.h index 5836f94a978..d84e3e63d33 100644 --- a/src/event/quic/ngx_event_quic_connection.h +++ b/src/event/quic/ngx_event_quic_connection.h @@ -230,6 +230,8 @@ struct ngx_quic_connection_s { ngx_event_t pto; ngx_event_t close; ngx_event_t path_validation; + ngx_event_t key_update; + ngx_msec_t last_cc; ngx_msec_t first_rtt; diff --git a/src/event/quic/ngx_event_quic_protection.c b/src/event/quic/ngx_event_quic_protection.c index ecac6f57876..5bc3c200f1e 100644 --- a/src/event/quic/ngx_event_quic_protection.c +++ b/src/event/quic/ngx_event_quic_protection.c @@ -700,23 +700,32 @@ ngx_quic_keys_switch(ngx_connection_t *c, ngx_quic_keys_t *keys) } -ngx_int_t -ngx_quic_keys_update(ngx_connection_t *c, ngx_quic_keys_t *keys) +void +ngx_quic_keys_update(ngx_event_t *ev) { - ngx_uint_t i; - ngx_quic_hkdf_t seq[6]; - ngx_quic_ciphers_t ciphers; - ngx_quic_secrets_t *current, *next; + ngx_uint_t i; + ngx_quic_hkdf_t seq[6]; + ngx_quic_keys_t *keys; + ngx_connection_t *c; + ngx_quic_ciphers_t ciphers; + ngx_quic_secrets_t *current, *next; + ngx_quic_connection_t *qc; + + c = ev->data; + qc = ngx_quic_get_connection(c); + keys = qc->keys; current = &keys->secrets[ssl_encryption_application]; next = &keys->next_key; ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic key update"); + c->log->action = "updating keys"; + if (ngx_quic_ciphers(keys->cipher, &ciphers, ssl_encryption_application) == NGX_ERROR) { - return NGX_ERROR; + goto failed; } next->client.secret.len = current->client.secret.len; @@ -744,11 +753,15 @@ ngx_quic_keys_update(ngx_connection_t *c, ngx_quic_keys_t *keys) for (i = 0; i < (sizeof(seq) / sizeof(seq[0])); i++) { if (ngx_quic_hkdf_expand(&seq[i], ciphers.d, c->log) != NGX_OK) { - return NGX_ERROR; + goto failed; } } - return NGX_OK; + return; + +failed: + + ngx_quic_close_connection(c, NGX_ERROR); } diff --git a/src/event/quic/ngx_event_quic_protection.h b/src/event/quic/ngx_event_quic_protection.h index 4e56ea9d1db..2d3006776ee 100644 --- a/src/event/quic/ngx_event_quic_protection.h +++ b/src/event/quic/ngx_event_quic_protection.h @@ -99,7 +99,7 @@ ngx_uint_t ngx_quic_keys_available(ngx_quic_keys_t *keys, void ngx_quic_keys_discard(ngx_quic_keys_t *keys, enum ssl_encryption_level_t level); void ngx_quic_keys_switch(ngx_connection_t *c, ngx_quic_keys_t *keys); -ngx_int_t ngx_quic_keys_update(ngx_connection_t *c, ngx_quic_keys_t *keys); +void ngx_quic_keys_update(ngx_event_t *ev); ngx_int_t ngx_quic_encrypt(ngx_quic_header_t *pkt, ngx_str_t *res); ngx_int_t ngx_quic_decrypt(ngx_quic_header_t *pkt, uint64_t *largest_pn); void ngx_quic_compute_nonce(u_char *nonce, size_t len, uint64_t pn); diff --git a/src/event/quic/ngx_event_quic_ssl.c b/src/event/quic/ngx_event_quic_ssl.c index e0862e29688..316d6b5eb73 100644 --- a/src/event/quic/ngx_event_quic_ssl.c +++ b/src/event/quic/ngx_event_quic_ssl.c @@ -482,9 +482,7 @@ ngx_quic_crypto_input(ngx_connection_t *c, ngx_chain_t *data) * Generating next keys before a key update is received. */ - if (ngx_quic_keys_update(c, qc->keys) != NGX_OK) { - return NGX_ERROR; - } + ngx_post_event(&qc->key_update, &ngx_posted_events); /* * RFC 9001, 4.9.2. Discarding Handshake Keys From 1bc204a3a564bcadbfb1a8a79f9e5a59d80669b3 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Wed, 30 Aug 2023 11:09:21 +0400 Subject: [PATCH 037/279] QUIC: use last client dcid to receive initial packets. Previously, original dcid was used to receive initial client packets in case server initial response was lost. However, last dcid should be used instead. These two are the same unless retry is used. In case of retry, client resends initial packet with a new dcid, that is different from the original dcid. If server response is lost, the client resends this packet again with the same dcid. This is shown in RFC 9000, 7.3. Authenticating Connection IDs, Figure 8. The issue manifested itself with creating multiple server sessions in response to each post-retry client initial packet, if server response is lost. --- src/event/quic/ngx_event_quic.c | 2 +- src/event/quic/ngx_event_quic_socket.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/event/quic/ngx_event_quic.c b/src/event/quic/ngx_event_quic.c index 2d7cecbc00d..4026540dbcf 100644 --- a/src/event/quic/ngx_event_quic.c +++ b/src/event/quic/ngx_event_quic.c @@ -1110,7 +1110,7 @@ ngx_quic_discard_ctx(ngx_connection_t *c, enum ssl_encryption_level_t level) } if (level == ssl_encryption_initial) { - /* close temporary listener with odcid */ + /* close temporary listener with initial dcid */ qsock = ngx_quic_find_socket(c, NGX_QUIC_UNSET_PN); if (qsock) { ngx_quic_close_socket(c, qsock); diff --git a/src/event/quic/ngx_event_quic_socket.c b/src/event/quic/ngx_event_quic_socket.c index 8339de3faef..c2bc822a55c 100644 --- a/src/event/quic/ngx_event_quic_socket.c +++ b/src/event/quic/ngx_event_quic_socket.c @@ -93,8 +93,8 @@ ngx_quic_open_sockets(ngx_connection_t *c, ngx_quic_connection_t *qc, tmp->sid.seqnum = NGX_QUIC_UNSET_PN; /* temporary socket */ - ngx_memcpy(tmp->sid.id, pkt->odcid.data, pkt->odcid.len); - tmp->sid.len = pkt->odcid.len; + ngx_memcpy(tmp->sid.id, pkt->dcid.data, pkt->dcid.len); + tmp->sid.len = pkt->dcid.len; if (ngx_quic_listen(c, qc, tmp) != NGX_OK) { goto failed; From ba30ff4c8de1ac8c79a12da50ef94e5d9f99fa66 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Thu, 31 Aug 2023 10:54:07 +0400 Subject: [PATCH 038/279] QUIC: ignore path validation socket error (ticket #2532). Previously, a socket error on a path being validated resulted in validation error and subsequent QUIC connection closure. Now the error is ignored and path validation proceeds as usual, with several retries and a timeout. When validating the old path after an apparent migration, that path may already be unavailable and sendmsg() may return an error, which should not result in QUIC connection close. When validating the new path, it's possible that the new client address is spoofed (See RFC 9000, 9.3.2. On-Path Address Spoofing). This address may as well be unavailable and should not trigger QUIC connection closure. --- src/event/quic/ngx_event_quic_migration.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/event/quic/ngx_event_quic_migration.c b/src/event/quic/ngx_event_quic_migration.c index 05b9a2863e1..bcec9af1d81 100644 --- a/src/event/quic/ngx_event_quic_migration.c +++ b/src/event/quic/ngx_event_quic_migration.c @@ -518,9 +518,7 @@ ngx_quic_validate_path(ngx_connection_t *c, ngx_quic_path_t *path) return NGX_ERROR; } - if (ngx_quic_send_path_challenge(c, path) != NGX_OK) { - return NGX_ERROR; - } + (void) ngx_quic_send_path_challenge(c, path); ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_application); pto = ngx_max(ngx_quic_pto(c, ctx), 1000); From fa46a5719924a5a4c48c515a903e0c46a4d98bcf Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Thu, 31 Aug 2023 22:59:17 +0300 Subject: [PATCH 039/279] Upstream: fixed handling of Status headers without reason-phrase. Status header with an empty reason-phrase, such as "Status: 404 ", is valid per CGI specification, but looses the trailing space during parsing. Currently, this results in "HTTP/1.1 404" HTTP status line in the response, which violates HTTP specification due to missing trailing space. With this change, only the status code is used from such short Status header lines, so nginx will generate status line itself, with the space and appropriate reason phrase if available. Reported at: https://mailman.nginx.org/pipermail/nginx/2023-August/EX7G4JUUHJWJE5UOAZMO5UD6OJILCYGX.html --- src/http/modules/ngx_http_fastcgi_module.c | 5 ++++- src/http/modules/ngx_http_scgi_module.c | 5 ++++- src/http/modules/ngx_http_uwsgi_module.c | 5 ++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/http/modules/ngx_http_fastcgi_module.c b/src/http/modules/ngx_http_fastcgi_module.c index 2d9a18f905f..b9890833d98 100644 --- a/src/http/modules/ngx_http_fastcgi_module.c +++ b/src/http/modules/ngx_http_fastcgi_module.c @@ -2048,7 +2048,10 @@ ngx_http_fastcgi_process_header(ngx_http_request_t *r) } u->headers_in.status_n = status; - u->headers_in.status_line = *status_line; + + if (status_line->len > 3) { + u->headers_in.status_line = *status_line; + } } else if (u->headers_in.location) { u->headers_in.status_n = 302; diff --git a/src/http/modules/ngx_http_scgi_module.c b/src/http/modules/ngx_http_scgi_module.c index 9fc18dcd30b..3acea87b764 100644 --- a/src/http/modules/ngx_http_scgi_module.c +++ b/src/http/modules/ngx_http_scgi_module.c @@ -1153,7 +1153,10 @@ ngx_http_scgi_process_header(ngx_http_request_t *r) } u->headers_in.status_n = status; - u->headers_in.status_line = *status_line; + + if (status_line->len > 3) { + u->headers_in.status_line = *status_line; + } } else if (u->headers_in.location) { u->headers_in.status_n = 302; diff --git a/src/http/modules/ngx_http_uwsgi_module.c b/src/http/modules/ngx_http_uwsgi_module.c index e4f721bb026..c1731ff484c 100644 --- a/src/http/modules/ngx_http_uwsgi_module.c +++ b/src/http/modules/ngx_http_uwsgi_module.c @@ -1381,7 +1381,10 @@ ngx_http_uwsgi_process_header(ngx_http_request_t *r) } u->headers_in.status_n = status; - u->headers_in.status_line = *status_line; + + if (status_line->len > 3) { + u->headers_in.status_line = *status_line; + } } else if (u->headers_in.location) { u->headers_in.status_n = 302; From 0d6ea58ebb9b589a1438930b6a64bd8ce1b52e62 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Fri, 1 Sep 2023 20:31:46 +0400 Subject: [PATCH 040/279] QUIC: refined sending CONNECTION_CLOSE in various packet types. As per RFC 9000, section 10.2.3, to ensure that peer successfully removed packet protection, CONNECTION_CLOSE can be sent in multiple packets using different packet protection levels. Now it is sent in all protection levels available. This roughly corresponds to the following paragraph: * Prior to confirming the handshake, a peer might be unable to process 1-RTT packets, so an endpoint SHOULD send a CONNECTION_CLOSE frame in both Handshake and 1-RTT packets. A server SHOULD also send a CONNECTION_CLOSE frame in an Initial packet. In practice, this change allows to avoid sending an Initial packet when we know the client has handshake keys, by checking if we have discarded initial keys. Also, this fixes sending CONNECTION_CLOSE when using QuicTLS with old QUIC API, where TLS stack releases application read keys before handshake confirmation; it is fixed by sending CONNECTION_CLOSE additionally in a Handshake packet. --- src/event/quic/ngx_event_quic.c | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/event/quic/ngx_event_quic.c b/src/event/quic/ngx_event_quic.c index 4026540dbcf..cd8beb35283 100644 --- a/src/event/quic/ngx_event_quic.c +++ b/src/event/quic/ngx_event_quic.c @@ -509,9 +509,6 @@ ngx_quic_close_connection(ngx_connection_t *c, ngx_int_t rc) * to terminate the connection immediately. */ - qc->error_level = c->ssl ? SSL_quic_read_level(c->ssl->connection) - : ssl_encryption_initial; - if (qc->error == (ngx_uint_t) -1) { qc->error = NGX_QUIC_ERR_INTERNAL_ERROR; qc->error_app = 0; @@ -524,17 +521,19 @@ ngx_quic_close_connection(ngx_connection_t *c, ngx_int_t rc) qc->error_app ? "app " : "", qc->error, qc->error_reason ? qc->error_reason : ""); - if (rc == NGX_OK) { - ctx = ngx_quic_get_send_ctx(qc, qc->error_level); - ngx_add_timer(&qc->close, 3 * ngx_quic_pto(c, ctx)); - } + for (i = 0; i < NGX_QUIC_SEND_CTX_LAST; i++) { + ctx = &qc->send_ctx[i]; - (void) ngx_quic_send_cc(c); + if (!ngx_quic_keys_available(qc->keys, ctx->level)) { + continue; + } - if (qc->error_level == ssl_encryption_handshake) { - /* for clients that might not have handshake keys */ - qc->error_level = ssl_encryption_initial; + qc->error_level = ctx->level; (void) ngx_quic_send_cc(c); + + if (rc == NGX_OK && !qc->close.timer_set) { + ngx_add_timer(&qc->close, 3 * ngx_quic_pto(c, ctx)); + } } } From b489ba83e9be446923facfe1a2fe392be3095d1f Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Fri, 1 Sep 2023 20:31:46 +0400 Subject: [PATCH 041/279] QUIC: removed use of SSL_quic_read_level and SSL_quic_write_level. As explained in BoringSSL change[1], levels were introduced in the original QUIC API to draw a line between when keys are released and when are active. In the new QUIC API they are released in separate calls when it's needed. BoringSSL has then a consideration to remove levels API, hence the change. If not available e.g. from a QUIC packet header, levels can be taken based on keys availability. The only real use of levels is to prevent using app keys before they are active in QuicTLS that provides the old BoringSSL QUIC API, it is replaced with an equivalent check of c->ssl->handshaked. This change also removes OpenSSL compat shims since they are no longer used. The only exception left is caching write level from the keylog callback in the internal field which is a handy equivalent of checking keys availability. [1] https://boringssl.googlesource.com/boringssl/+/1e859054 --- src/event/quic/ngx_event_quic.c | 5 +--- .../quic/ngx_event_quic_openssl_compat.c | 28 ------------------- .../quic/ngx_event_quic_openssl_compat.h | 2 -- src/event/quic/ngx_event_quic_ssl.c | 24 +++++----------- 4 files changed, 8 insertions(+), 51 deletions(-) diff --git a/src/event/quic/ngx_event_quic.c b/src/event/quic/ngx_event_quic.c index cd8beb35283..6852bb070aa 100644 --- a/src/event/quic/ngx_event_quic.c +++ b/src/event/quic/ngx_event_quic.c @@ -963,10 +963,7 @@ ngx_quic_handle_payload(ngx_connection_t *c, ngx_quic_header_t *pkt) #if !defined (OPENSSL_IS_BORINGSSL) /* OpenSSL provides read keys for an application level before it's ready */ - if (pkt->level == ssl_encryption_application - && SSL_quic_read_level(c->ssl->connection) - < ssl_encryption_application) - { + if (pkt->level == ssl_encryption_application && !c->ssl->handshaked) { ngx_log_error(NGX_LOG_INFO, c->log, 0, "quic no %s keys ready, ignoring packet", ngx_quic_level_name(pkt->level)); diff --git a/src/event/quic/ngx_event_quic_openssl_compat.c b/src/event/quic/ngx_event_quic_openssl_compat.c index 318feda10f3..e970cfb9be7 100644 --- a/src/event/quic/ngx_event_quic_openssl_compat.c +++ b/src/event/quic/ngx_event_quic_openssl_compat.c @@ -44,7 +44,6 @@ struct ngx_quic_compat_s { const SSL_QUIC_METHOD *method; enum ssl_encryption_level_t write_level; - enum ssl_encryption_level_t read_level; uint64_t read_record; ngx_quic_compat_keys_t keys; @@ -213,7 +212,6 @@ ngx_quic_compat_keylog_callback(const SSL *ssl, const char *line) } else { com->method->set_read_secret((SSL *) ssl, level, cipher, secret, n); - com->read_level = level; com->read_record = 0; (void) ngx_quic_compat_set_encryption_secret(c->log, &com->keys, level, @@ -583,32 +581,6 @@ ngx_quic_compat_create_record(ngx_quic_compat_record_t *rec, ngx_str_t *res) } -enum ssl_encryption_level_t -SSL_quic_read_level(const SSL *ssl) -{ - ngx_connection_t *c; - ngx_quic_connection_t *qc; - - c = ngx_ssl_get_connection(ssl); - qc = ngx_quic_get_connection(c); - - return qc->compat->read_level; -} - - -enum ssl_encryption_level_t -SSL_quic_write_level(const SSL *ssl) -{ - ngx_connection_t *c; - ngx_quic_connection_t *qc; - - c = ngx_ssl_get_connection(ssl); - qc = ngx_quic_get_connection(c); - - return qc->compat->write_level; -} - - int SSL_set_quic_transport_params(SSL *ssl, const uint8_t *params, size_t params_len) diff --git a/src/event/quic/ngx_event_quic_openssl_compat.h b/src/event/quic/ngx_event_quic_openssl_compat.h index b04f6e0b5c7..77cc3cb0d83 100644 --- a/src/event/quic/ngx_event_quic_openssl_compat.h +++ b/src/event/quic/ngx_event_quic_openssl_compat.h @@ -48,8 +48,6 @@ ngx_int_t ngx_quic_compat_init(ngx_conf_t *cf, SSL_CTX *ctx); int SSL_set_quic_method(SSL *ssl, const SSL_QUIC_METHOD *quic_method); int SSL_provide_quic_data(SSL *ssl, enum ssl_encryption_level_t level, const uint8_t *data, size_t len); -enum ssl_encryption_level_t SSL_quic_read_level(const SSL *ssl); -enum ssl_encryption_level_t SSL_quic_write_level(const SSL *ssl); int SSL_set_quic_transport_params(SSL *ssl, const uint8_t *params, size_t params_len); void SSL_get_peer_quic_transport_params(const SSL *ssl, diff --git a/src/event/quic/ngx_event_quic_ssl.c b/src/event/quic/ngx_event_quic_ssl.c index 316d6b5eb73..c719a1dd4ae 100644 --- a/src/event/quic/ngx_event_quic_ssl.c +++ b/src/event/quic/ngx_event_quic_ssl.c @@ -43,7 +43,8 @@ static int ngx_quic_add_handshake_data(ngx_ssl_conn_t *ssl_conn, static int ngx_quic_flush_flight(ngx_ssl_conn_t *ssl_conn); static int ngx_quic_send_alert(ngx_ssl_conn_t *ssl_conn, enum ssl_encryption_level_t level, uint8_t alert); -static ngx_int_t ngx_quic_crypto_input(ngx_connection_t *c, ngx_chain_t *data); +static ngx_int_t ngx_quic_crypto_input(ngx_connection_t *c, ngx_chain_t *data, + enum ssl_encryption_level_t level); #if (NGX_QUIC_BORINGSSL_API) @@ -354,7 +355,7 @@ ngx_quic_handle_crypto_frame(ngx_connection_t *c, ngx_quic_header_t *pkt, } if (f->offset == ctx->crypto.offset) { - if (ngx_quic_crypto_input(c, frame->data) != NGX_OK) { + if (ngx_quic_crypto_input(c, frame->data, pkt->level) != NGX_OK) { return NGX_ERROR; } @@ -372,7 +373,7 @@ ngx_quic_handle_crypto_frame(ngx_connection_t *c, ngx_quic_header_t *pkt, cl = ngx_quic_read_buffer(c, &ctx->crypto, (uint64_t) -1); if (cl) { - if (ngx_quic_crypto_input(c, cl) != NGX_OK) { + if (ngx_quic_crypto_input(c, cl, pkt->level) != NGX_OK) { return NGX_ERROR; } @@ -384,7 +385,8 @@ ngx_quic_handle_crypto_frame(ngx_connection_t *c, ngx_quic_header_t *pkt, static ngx_int_t -ngx_quic_crypto_input(ngx_connection_t *c, ngx_chain_t *data) +ngx_quic_crypto_input(ngx_connection_t *c, ngx_chain_t *data, + enum ssl_encryption_level_t level) { int n, sslerr; ngx_buf_t *b; @@ -397,17 +399,10 @@ ngx_quic_crypto_input(ngx_connection_t *c, ngx_chain_t *data) ssl_conn = c->ssl->connection; - ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, - "quic SSL_quic_read_level:%d SSL_quic_write_level:%d", - (int) SSL_quic_read_level(ssl_conn), - (int) SSL_quic_write_level(ssl_conn)); - for (cl = data; cl; cl = cl->next) { b = cl->buf; - if (!SSL_provide_quic_data(ssl_conn, SSL_quic_read_level(ssl_conn), - b->pos, b->last - b->pos)) - { + if (!SSL_provide_quic_data(ssl_conn, level, b->pos, b->last - b->pos)) { ngx_ssl_error(NGX_LOG_INFO, c->log, 0, "SSL_provide_quic_data() failed"); return NGX_ERROR; @@ -416,11 +411,6 @@ ngx_quic_crypto_input(ngx_connection_t *c, ngx_chain_t *data) n = SSL_do_handshake(ssl_conn); - ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, - "quic SSL_quic_read_level:%d SSL_quic_write_level:%d", - (int) SSL_quic_read_level(ssl_conn), - (int) SSL_quic_write_level(ssl_conn)); - ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, "SSL_do_handshake: %d", n); if (n <= 0) { From 33dca887925d9cec7153252dcd5380c394090936 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Wed, 13 Sep 2023 17:59:37 +0400 Subject: [PATCH 042/279] QUIC: "handshake_timeout" configuration parameter. Previously QUIC did not have such parameter and handshake duration was controlled by HTTP/3. However that required creating and storing HTTP/3 session on first client datagram. Apparently there's no convenient way to store the session object until QUIC handshake is complete. In the followup patches session creation will be postponed to init() callback. --- src/event/quic/ngx_event_quic.c | 6 ++++++ src/event/quic/ngx_event_quic.h | 3 ++- src/event/quic/ngx_event_quic_streams.c | 4 ++++ src/event/quic/ngx_event_quic_transport.c | 2 +- src/http/v3/ngx_http_v3_module.c | 8 ++++++-- src/http/v3/ngx_http_v3_request.c | 8 +------- 6 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/event/quic/ngx_event_quic.c b/src/event/quic/ngx_event_quic.c index 6852bb070aa..0032a550566 100644 --- a/src/event/quic/ngx_event_quic.c +++ b/src/event/quic/ngx_event_quic.c @@ -211,6 +211,8 @@ ngx_quic_run(ngx_connection_t *c, ngx_quic_conf_t *conf) qc = ngx_quic_get_connection(c); ngx_add_timer(c->read, qc->tp.max_idle_timeout); + ngx_add_timer(&qc->close, qc->conf->handshake_timeout); + ngx_quic_connstate_dbg(c); c->read->handler = ngx_quic_input_handler; @@ -485,6 +487,10 @@ ngx_quic_close_connection(ngx_connection_t *c, ngx_int_t rc) ngx_quic_free_frames(c, &qc->send_ctx[i].sent); } + if (qc->close.timer_set) { + ngx_del_timer(&qc->close); + } + if (rc == NGX_DONE) { /* diff --git a/src/event/quic/ngx_event_quic.h b/src/event/quic/ngx_event_quic.h index ca15200b43f..15201671d4d 100644 --- a/src/event/quic/ngx_event_quic.h +++ b/src/event/quic/ngx_event_quic.h @@ -67,7 +67,8 @@ typedef struct { ngx_flag_t retry; ngx_flag_t gso_enabled; ngx_flag_t disable_active_migration; - ngx_msec_t timeout; + ngx_msec_t handshake_timeout; + ngx_msec_t idle_timeout; ngx_str_t host_key; size_t stream_buffer_size; ngx_uint_t max_concurrent_streams_bidi; diff --git a/src/event/quic/ngx_event_quic_streams.c b/src/event/quic/ngx_event_quic_streams.c index b4b0eda0527..df04d0f0740 100644 --- a/src/event/quic/ngx_event_quic_streams.c +++ b/src/event/quic/ngx_event_quic_streams.c @@ -630,6 +630,10 @@ ngx_quic_do_init_streams(ngx_connection_t *c) qc->streams.initialized = 1; + if (!qc->closing && qc->close.timer_set) { + ngx_del_timer(&qc->close); + } + return NGX_OK; } diff --git a/src/event/quic/ngx_event_quic_transport.c b/src/event/quic/ngx_event_quic_transport.c index fafe85f41bd..4e0324f4a98 100644 --- a/src/event/quic/ngx_event_quic_transport.c +++ b/src/event/quic/ngx_event_quic_transport.c @@ -1985,7 +1985,7 @@ ngx_quic_init_transport_params(ngx_quic_tp_t *tp, ngx_quic_conf_t *qcf) * tp->preferred_address = NULL */ - tp->max_idle_timeout = qcf->timeout; + tp->max_idle_timeout = qcf->idle_timeout; tp->max_udp_payload_size = NGX_QUIC_MAX_UDP_PAYLOAD_SIZE; diff --git a/src/http/v3/ngx_http_v3_module.c b/src/http/v3/ngx_http_v3_module.c index 875e5f29ba0..139bd65f3d9 100644 --- a/src/http/v3/ngx_http_v3_module.c +++ b/src/http/v3/ngx_http_v3_module.c @@ -192,7 +192,7 @@ ngx_http_v3_create_srv_conf(ngx_conf_t *cf) * h3scf->quic.host_key = { 0, NULL } * h3scf->quic.stream_reject_code_uni = 0; * h3scf->quic.disable_active_migration = 0; - * h3scf->quic.timeout = 0; + * h3scf->quic.idle_timeout = 0; * h3scf->max_blocked_streams = 0; */ @@ -223,7 +223,8 @@ ngx_http_v3_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) ngx_http_v3_srv_conf_t *prev = parent; ngx_http_v3_srv_conf_t *conf = child; - ngx_http_ssl_srv_conf_t *sscf; + ngx_http_ssl_srv_conf_t *sscf; + ngx_http_core_srv_conf_t *cscf; ngx_conf_merge_value(conf->enable, prev->enable, 1); @@ -281,6 +282,9 @@ ngx_http_v3_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) return NGX_CONF_ERROR; } + cscf = ngx_http_conf_get_module_srv_conf(cf, ngx_http_core_module); + conf->quic.handshake_timeout = cscf->client_header_timeout; + sscf = ngx_http_conf_get_module_srv_conf(cf, ngx_http_ssl_module); conf->quic.ssl = &sscf->ssl; diff --git a/src/http/v3/ngx_http_v3_request.c b/src/http/v3/ngx_http_v3_request.c index 6f72dc40223..2f5503abace 100644 --- a/src/http/v3/ngx_http_v3_request.c +++ b/src/http/v3/ngx_http_v3_request.c @@ -58,18 +58,15 @@ static const struct { void ngx_http_v3_init_stream(ngx_connection_t *c) { - ngx_http_v3_session_t *h3c; ngx_http_connection_t *hc, *phc; ngx_http_v3_srv_conf_t *h3scf; ngx_http_core_loc_conf_t *clcf; - ngx_http_core_srv_conf_t *cscf; hc = c->data; hc->ssl = 1; clcf = ngx_http_get_module_loc_conf(hc->conf_ctx, ngx_http_core_module); - cscf = ngx_http_get_module_srv_conf(hc->conf_ctx, ngx_http_core_module); h3scf = ngx_http_get_module_srv_conf(hc->conf_ctx, ngx_http_v3_module); if (c->quic == NULL) { @@ -78,10 +75,7 @@ ngx_http_v3_init_stream(ngx_connection_t *c) return; } - h3c = hc->v3_session; - ngx_add_timer(&h3c->keepalive, cscf->client_header_timeout); - - h3scf->quic.timeout = clcf->keepalive_timeout; + h3scf->quic.idle_timeout = clcf->keepalive_timeout; ngx_quic_run(c, &h3scf->quic); return; } From ec37134416d4fd98d8cb8f02776a711c50398684 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Wed, 13 Sep 2023 17:57:13 +0400 Subject: [PATCH 043/279] HTTP/3: moved variable initialization. --- src/http/v3/ngx_http_v3_request.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/http/v3/ngx_http_v3_request.c b/src/http/v3/ngx_http_v3_request.c index 2f5503abace..92c204c24b0 100644 --- a/src/http/v3/ngx_http_v3_request.c +++ b/src/http/v3/ngx_http_v3_request.c @@ -67,7 +67,6 @@ ngx_http_v3_init_stream(ngx_connection_t *c) hc->ssl = 1; clcf = ngx_http_get_module_loc_conf(hc->conf_ctx, ngx_http_core_module); - h3scf = ngx_http_get_module_srv_conf(hc->conf_ctx, ngx_http_v3_module); if (c->quic == NULL) { if (ngx_http_v3_init_session(c) != NGX_OK) { @@ -75,7 +74,9 @@ ngx_http_v3_init_stream(ngx_connection_t *c) return; } + h3scf = ngx_http_get_module_srv_conf(hc->conf_ctx, ngx_http_v3_module); h3scf->quic.idle_timeout = clcf->keepalive_timeout; + ngx_quic_run(c, &h3scf->quic); return; } From 6ecf576e34c3780b6cdb3f509c89042aa411e3d2 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Thu, 21 Sep 2023 19:32:38 +0400 Subject: [PATCH 044/279] QUIC: do not call shutdown() when handshake is in progress. Instead, when worker is shutting down and handshake is not yet completed, connection is terminated immediately. Previously the callback could be called while QUIC handshake was in progress and, what's more important, before the init() callback. Now it's postponed after init(). This change is a preparation to postponing HTTP/3 session creation to init(). --- src/event/quic/ngx_event_quic.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/event/quic/ngx_event_quic.c b/src/event/quic/ngx_event_quic.c index 0032a550566..b23434c0edf 100644 --- a/src/event/quic/ngx_event_quic.c +++ b/src/event/quic/ngx_event_quic.c @@ -420,7 +420,7 @@ ngx_quic_input_handler(ngx_event_t *rev) if (c->close) { c->close = 0; - if (!ngx_exiting) { + if (!ngx_exiting || !qc->streams.initialized) { qc->error = NGX_QUIC_ERR_NO_ERROR; qc->error_reason = "graceful shutdown"; ngx_quic_close_connection(c, NGX_ERROR); From 26e606a6bcdfa4001bfb6bd24612e8aafa6513b2 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Thu, 14 Sep 2023 14:13:43 +0400 Subject: [PATCH 045/279] HTTP/3: postponed session creation to init() callback. Now the session object is assigned to c->data while ngx_http_connection_t object is referenced by its http_connection field, similar to ngx_http_v2_connection_t and ngx_http_request_t. The change allows to eliminate v3_session field from ngx_http_connection_t. The field was under NGX_HTTP_V3 macro, which was a source of binary compatibility problems when nginx/module is build with/without HTTP/3 support. Postponing is essential since c->data should retain the reference to ngx_http_connection_t object throughout QUIC handshake, because SSL callbacks ngx_http_ssl_servername() and ngx_http_ssl_alpn_select() rely on this. --- src/http/ngx_http_request.h | 4 ---- src/http/v3/ngx_http_v3.c | 4 +++- src/http/v3/ngx_http_v3.h | 9 ++++++--- src/http/v3/ngx_http_v3_request.c | 9 ++++----- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/http/ngx_http_request.h b/src/http/ngx_http_request.h index 92c54461941..cc3b7c0acba 100644 --- a/src/http/ngx_http_request.h +++ b/src/http/ngx_http_request.h @@ -324,10 +324,6 @@ typedef struct { #endif #endif -#if (NGX_HTTP_V3 || NGX_COMPAT) - ngx_http_v3_session_t *v3_session; -#endif - ngx_chain_t *busy; ngx_int_t nbusy; diff --git a/src/http/v3/ngx_http_v3.c b/src/http/v3/ngx_http_v3.c index eb86b2da584..8db229b29aa 100644 --- a/src/http/v3/ngx_http_v3.c +++ b/src/http/v3/ngx_http_v3.c @@ -30,6 +30,8 @@ ngx_http_v3_init_session(ngx_connection_t *c) goto failed; } + h3c->http_connection = hc; + ngx_queue_init(&h3c->blocked); h3c->keepalive.log = c->log; @@ -48,7 +50,7 @@ ngx_http_v3_init_session(ngx_connection_t *c) cln->handler = ngx_http_v3_cleanup_session; cln->data = h3c; - hc->v3_session = h3c; + c->data = h3c; return NGX_OK; diff --git a/src/http/v3/ngx_http_v3.h b/src/http/v3/ngx_http_v3.h index 94b0d3e78c3..9dcb5e6a7dd 100644 --- a/src/http/v3/ngx_http_v3.h +++ b/src/http/v3/ngx_http_v3.h @@ -78,11 +78,12 @@ #define NGX_HTTP_V3_ERR_DECODER_STREAM_ERROR 0x202 -#define ngx_http_quic_get_connection(c) \ - ((ngx_http_connection_t *) ((c)->quic ? (c)->quic->parent->data \ +#define ngx_http_v3_get_session(c) \ + ((ngx_http_v3_session_t *) ((c)->quic ? (c)->quic->parent->data \ : (c)->data)) -#define ngx_http_v3_get_session(c) ngx_http_quic_get_connection(c)->v3_session +#define ngx_http_quic_get_connection(c) \ + (ngx_http_v3_get_session(c)->http_connection) #define ngx_http_v3_get_module_loc_conf(c, module) \ ngx_http_get_module_loc_conf(ngx_http_quic_get_connection(c)->conf_ctx, \ @@ -120,6 +121,8 @@ struct ngx_http_v3_parse_s { struct ngx_http_v3_session_s { + ngx_http_connection_t *http_connection; + ngx_http_v3_dynamic_table_t table; ngx_event_t keepalive; diff --git a/src/http/v3/ngx_http_v3_request.c b/src/http/v3/ngx_http_v3_request.c index 92c204c24b0..87f5f3214cd 100644 --- a/src/http/v3/ngx_http_v3_request.c +++ b/src/http/v3/ngx_http_v3_request.c @@ -69,11 +69,6 @@ ngx_http_v3_init_stream(ngx_connection_t *c) clcf = ngx_http_get_module_loc_conf(hc->conf_ctx, ngx_http_core_module); if (c->quic == NULL) { - if (ngx_http_v3_init_session(c) != NGX_OK) { - ngx_http_close_connection(c); - return; - } - h3scf = ngx_http_get_module_srv_conf(hc->conf_ctx, ngx_http_v3_module); h3scf->quic.idle_timeout = clcf->keepalive_timeout; @@ -113,6 +108,10 @@ ngx_http_v3_init(ngx_connection_t *c) ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "http3 init"); + if (ngx_http_v3_init_session(c) != NGX_OK) { + return NGX_ERROR; + } + h3c = ngx_http_v3_get_session(c); clcf = ngx_http_v3_get_module_loc_conf(c, ngx_http_core_module); ngx_add_timer(&h3c->keepalive, clcf->keepalive_timeout); From 196289ac18b94c6bb0fa4f5e9a33f12757444399 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Thu, 14 Sep 2023 14:15:20 +0400 Subject: [PATCH 046/279] QUIC: simplified setting close timer when closing connection. Previously, the timer was never reset due to an explicit check. The check was added in 36b59521a41c as part of connection close simplification. The reason was to retain the earliest timeout. However, the timeouts are all the same while QUIC handshake is in progress and resetting the timer for the same value has no performance implications. After handshake completion there's only application level. The change removes the check. --- src/event/quic/ngx_event_quic.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/event/quic/ngx_event_quic.c b/src/event/quic/ngx_event_quic.c index b23434c0edf..df3833e91e1 100644 --- a/src/event/quic/ngx_event_quic.c +++ b/src/event/quic/ngx_event_quic.c @@ -537,7 +537,7 @@ ngx_quic_close_connection(ngx_connection_t *c, ngx_int_t rc) qc->error_level = ctx->level; (void) ngx_quic_send_cc(c); - if (rc == NGX_OK && !qc->close.timer_set) { + if (rc == NGX_OK) { ngx_add_timer(&qc->close, 3 * ngx_quic_pto(c, ctx)); } } From 027b68168867514665751a65780f050952bc48ec Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Wed, 13 Sep 2023 17:48:15 +0400 Subject: [PATCH 047/279] Modules compatibility: added QUIC to signature (ticket #2539). Enabling QUIC changes ngx_connection_t layout, which is why it should be added to the signature. --- src/core/ngx_module.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/core/ngx_module.h b/src/core/ngx_module.h index 6fb455426b3..a415cd6d9e8 100644 --- a/src/core/ngx_module.h +++ b/src/core/ngx_module.h @@ -107,7 +107,12 @@ #endif #define NGX_MODULE_SIGNATURE_17 "0" + +#if (NGX_QUIC || NGX_COMPAT) +#define NGX_MODULE_SIGNATURE_18 "1" +#else #define NGX_MODULE_SIGNATURE_18 "0" +#endif #if (NGX_HAVE_OPENAT) #define NGX_MODULE_SIGNATURE_19 "1" From c37fdcdd1e1527d2c98cc68a978cf928589c7330 Mon Sep 17 00:00:00 2001 From: Vladimir Khomutov Date: Fri, 22 Sep 2023 19:23:57 +0400 Subject: [PATCH 048/279] QUIC: handle callback errors in compat. The error may be triggered in add_handhshake_data() by incorrect transport parameter sent by client. The expected behaviour in this case is to close connection complaining about incorrect parameter. Currently the connection just times out. --- src/event/quic/ngx_event_quic_openssl_compat.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/event/quic/ngx_event_quic_openssl_compat.c b/src/event/quic/ngx_event_quic_openssl_compat.c index e970cfb9be7..cc6b95d482d 100644 --- a/src/event/quic/ngx_event_quic_openssl_compat.c +++ b/src/event/quic/ngx_event_quic_openssl_compat.c @@ -408,7 +408,9 @@ ngx_quic_compat_message_callback(int write_p, int version, int content_type, "quic compat tx %s len:%uz ", ngx_quic_level_name(level), len); - (void) com->method->add_handshake_data(ssl, level, buf, len); + if (com->method->add_handshake_data(ssl, level, buf, len) != 1) { + goto failed; + } break; @@ -420,11 +422,19 @@ ngx_quic_compat_message_callback(int write_p, int version, int content_type, "quic compat %s alert:%ui len:%uz ", ngx_quic_level_name(level), alert, len); - (void) com->method->send_alert(ssl, level, alert); + if (com->method->send_alert(ssl, level, alert) != 1) { + goto failed; + } } break; } + + return; + +failed: + + ngx_post_event(&qc->close, &ngx_posted_events); } From 6ceef192e7af1c507826ac38a2d43f08bf265fb9 Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Tue, 10 Oct 2023 15:13:39 +0300 Subject: [PATCH 049/279] HTTP/2: per-iteration stream handling limit. To ensure that attempts to flood servers with many streams are detected early, a limit of no more than 2 * max_concurrent_streams new streams per one event loop iteration was introduced. This limit is applied even if max_concurrent_streams is not yet reached - for example, if corresponding streams are handled synchronously or reset. Further, refused streams are now limited to maximum of max_concurrent_streams and 100, similarly to priority_limit initial value, providing some tolerance to clients trying to open several streams at the connection start, yet low tolerance to flooding attempts. --- src/http/v2/ngx_http_v2.c | 15 +++++++++++++++ src/http/v2/ngx_http_v2.h | 2 ++ 2 files changed, 17 insertions(+) diff --git a/src/http/v2/ngx_http_v2.c b/src/http/v2/ngx_http_v2.c index 7c05ff1e78b..410a8be24eb 100644 --- a/src/http/v2/ngx_http_v2.c +++ b/src/http/v2/ngx_http_v2.c @@ -347,6 +347,7 @@ ngx_http_v2_read_handler(ngx_event_t *rev) ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "http2 read handler"); h2c->blocked = 1; + h2c->new_streams = 0; if (c->close) { c->close = 0; @@ -1284,6 +1285,14 @@ ngx_http_v2_state_headers(ngx_http_v2_connection_t *h2c, u_char *pos, goto rst_stream; } + if (h2c->new_streams++ >= 2 * h2scf->concurrent_streams) { + ngx_log_error(NGX_LOG_INFO, h2c->connection->log, 0, + "client sent too many streams at once"); + + status = NGX_HTTP_V2_REFUSED_STREAM; + goto rst_stream; + } + if (!h2c->settings_ack && !(h2c->state.flags & NGX_HTTP_V2_END_STREAM_FLAG) && h2scf->preread_size < NGX_HTTP_V2_DEFAULT_WINDOW) @@ -1349,6 +1358,12 @@ ngx_http_v2_state_headers(ngx_http_v2_connection_t *h2c, u_char *pos, rst_stream: + if (h2c->refused_streams++ > ngx_max(h2scf->concurrent_streams, 100)) { + ngx_log_error(NGX_LOG_INFO, h2c->connection->log, 0, + "client sent too many refused streams"); + return ngx_http_v2_connection_error(h2c, NGX_HTTP_V2_NO_ERROR); + } + if (ngx_http_v2_send_rst_stream(h2c, h2c->state.sid, status) != NGX_OK) { return ngx_http_v2_connection_error(h2c, NGX_HTTP_V2_INTERNAL_ERROR); } diff --git a/src/http/v2/ngx_http_v2.h b/src/http/v2/ngx_http_v2.h index cb9014ccfb1..6751b3026d8 100644 --- a/src/http/v2/ngx_http_v2.h +++ b/src/http/v2/ngx_http_v2.h @@ -131,6 +131,8 @@ struct ngx_http_v2_connection_s { ngx_uint_t processing; ngx_uint_t frames; ngx_uint_t idle; + ngx_uint_t new_streams; + ngx_uint_t refused_streams; ngx_uint_t priority_limit; size_t send_window; From 284a0c73771e3a2c57af6e74d96d9a6878b2e7b4 Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Tue, 17 Oct 2023 02:39:38 +0300 Subject: [PATCH 050/279] Core: fixed memory leak on configuration reload with PCRE2. In ngx_regex_cleanup() allocator wasn't configured when calling pcre2_compile_context_free() and pcre2_match_data_free(), resulting in no ngx_free() call and leaked memory. Fix is ensure that allocator is configured for global allocations, so that ngx_free() is actually called to free memory. Additionally, ngx_regex_compile_context was cleared in ngx_regex_module_init(). It should be either not cleared, so it will be freed by ngx_regex_cleanup(), or properly freed. Fix is to not clear it, so ngx_regex_cleanup() will be able to free it. Reported by ZhenZhong Wu, https://mailman.nginx.org/pipermail/nginx-devel/2023-September/3Z5FIKUDRN2WBSL3JWTZJ7SXDA6YIWPB.html --- src/core/ngx_regex.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/core/ngx_regex.c b/src/core/ngx_regex.c index 91381f49942..5b13c5db389 100644 --- a/src/core/ngx_regex.c +++ b/src/core/ngx_regex.c @@ -600,6 +600,8 @@ ngx_regex_cleanup(void *data) * the new cycle, these will be re-allocated. */ + ngx_regex_malloc_init(NULL); + if (ngx_regex_compile_context) { pcre2_compile_context_free(ngx_regex_compile_context); ngx_regex_compile_context = NULL; @@ -611,6 +613,8 @@ ngx_regex_cleanup(void *data) ngx_regex_match_data_size = 0; } + ngx_regex_malloc_done(); + #endif } @@ -706,9 +710,6 @@ ngx_regex_module_init(ngx_cycle_t *cycle) ngx_regex_malloc_done(); ngx_regex_studies = NULL; -#if (NGX_PCRE2) - ngx_regex_compile_context = NULL; -#endif return NGX_OK; } From c93cb45ae30760b7cd4ce2d9e053a36449d4e233 Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Wed, 18 Oct 2023 04:30:11 +0300 Subject: [PATCH 051/279] Core: changed ngx_queue_sort() to use merge sort. This improves nginx startup times significantly when using very large number of locations due to computational complexity of the sorting algorithm being used: insertion sort is O(n*n) on average, while merge sort is O(n*log(n)). In particular, in a test configuration with 20k locations total startup time is reduced from 8 seconds to 0.9 seconds. Prodded by Yusuke Nojima, https://mailman.nginx.org/pipermail/nginx-devel/2023-September/NUL3Y2FPPFSHMPTFTL65KXSXNTX3NQMK.html --- src/core/ngx_queue.c | 52 +++++++++++++++++++++++++++++++++----------- src/core/ngx_queue.h | 3 +++ 2 files changed, 42 insertions(+), 13 deletions(-) diff --git a/src/core/ngx_queue.c b/src/core/ngx_queue.c index 3cacaf3a883..3d1d5898849 100644 --- a/src/core/ngx_queue.c +++ b/src/core/ngx_queue.c @@ -9,6 +9,10 @@ #include +static void ngx_queue_merge(ngx_queue_t *queue, ngx_queue_t *tail, + ngx_int_t (*cmp)(const ngx_queue_t *, const ngx_queue_t *)); + + /* * find the middle queue element if the queue has odd number of elements * or the first element of the queue's second part otherwise @@ -45,13 +49,13 @@ ngx_queue_middle(ngx_queue_t *queue) } -/* the stable insertion sort */ +/* the stable merge sort */ void ngx_queue_sort(ngx_queue_t *queue, ngx_int_t (*cmp)(const ngx_queue_t *, const ngx_queue_t *)) { - ngx_queue_t *q, *prev, *next; + ngx_queue_t *q, tail; q = ngx_queue_head(queue); @@ -59,22 +63,44 @@ ngx_queue_sort(ngx_queue_t *queue, return; } - for (q = ngx_queue_next(q); q != ngx_queue_sentinel(queue); q = next) { + q = ngx_queue_middle(queue); + + ngx_queue_split(queue, q, &tail); + + ngx_queue_sort(queue, cmp); + ngx_queue_sort(&tail, cmp); + + ngx_queue_merge(queue, &tail, cmp); +} + - prev = ngx_queue_prev(q); - next = ngx_queue_next(q); +static void +ngx_queue_merge(ngx_queue_t *queue, ngx_queue_t *tail, + ngx_int_t (*cmp)(const ngx_queue_t *, const ngx_queue_t *)) +{ + ngx_queue_t *q1, *q2; - ngx_queue_remove(q); + q1 = ngx_queue_head(queue); + q2 = ngx_queue_head(tail); - do { - if (cmp(prev, q) <= 0) { - break; - } + for ( ;; ) { + if (q1 == ngx_queue_sentinel(queue)) { + ngx_queue_add(queue, tail); + break; + } - prev = ngx_queue_prev(prev); + if (q2 == ngx_queue_sentinel(tail)) { + break; + } + + if (cmp(q1, q2) <= 0) { + q1 = ngx_queue_next(q1); + continue; + } - } while (prev != ngx_queue_sentinel(queue)); + ngx_queue_remove(q2); + ngx_queue_insert_before(q1, q2); - ngx_queue_insert_after(prev, q); + q2 = ngx_queue_head(tail); } } diff --git a/src/core/ngx_queue.h b/src/core/ngx_queue.h index 038bf121138..0f82f173e4d 100644 --- a/src/core/ngx_queue.h +++ b/src/core/ngx_queue.h @@ -47,6 +47,9 @@ struct ngx_queue_s { (h)->prev = x +#define ngx_queue_insert_before ngx_queue_insert_tail + + #define ngx_queue_head(h) \ (h)->next From cd5f4cd8d3991414167a734f452e8e3fe98f3916 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Thu, 31 Aug 2023 19:54:10 +0400 Subject: [PATCH 052/279] QUIC: split keys availability checks to read and write sides. Keys may be released by TLS stack in different times, so it makes sense to check this independently as well. This allows to fine-tune what key direction is used when checking keys availability. When discarding, server keys are now marked in addition to client keys. --- src/event/quic/ngx_event_quic.c | 8 +++++--- src/event/quic/ngx_event_quic_protection.c | 9 +++++++-- src/event/quic/ngx_event_quic_protection.h | 2 +- src/event/quic/ngx_event_quic_ssl.c | 2 +- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/event/quic/ngx_event_quic.c b/src/event/quic/ngx_event_quic.c index df3833e91e1..c9cd527a153 100644 --- a/src/event/quic/ngx_event_quic.c +++ b/src/event/quic/ngx_event_quic.c @@ -530,7 +530,7 @@ ngx_quic_close_connection(ngx_connection_t *c, ngx_int_t rc) for (i = 0; i < NGX_QUIC_SEND_CTX_LAST; i++) { ctx = &qc->send_ctx[i]; - if (!ngx_quic_keys_available(qc->keys, ctx->level)) { + if (!ngx_quic_keys_available(qc->keys, ctx->level, 1)) { continue; } @@ -959,7 +959,7 @@ ngx_quic_handle_payload(ngx_connection_t *c, ngx_quic_header_t *pkt) c->log->action = "decrypting packet"; - if (!ngx_quic_keys_available(qc->keys, pkt->level)) { + if (!ngx_quic_keys_available(qc->keys, pkt->level, 0)) { ngx_log_error(NGX_LOG_INFO, c->log, 0, "quic no %s keys, ignoring packet", ngx_quic_level_name(pkt->level)); @@ -1082,7 +1082,9 @@ ngx_quic_discard_ctx(ngx_connection_t *c, enum ssl_encryption_level_t level) qc = ngx_quic_get_connection(c); - if (!ngx_quic_keys_available(qc->keys, level)) { + if (!ngx_quic_keys_available(qc->keys, level, 0) + && !ngx_quic_keys_available(qc->keys, level, 1)) + { return; } diff --git a/src/event/quic/ngx_event_quic_protection.c b/src/event/quic/ngx_event_quic_protection.c index 5bc3c200f1e..9f8169988a6 100644 --- a/src/event/quic/ngx_event_quic_protection.c +++ b/src/event/quic/ngx_event_quic_protection.c @@ -672,9 +672,13 @@ ngx_quic_keys_set_encryption_secret(ngx_log_t *log, ngx_uint_t is_write, ngx_uint_t ngx_quic_keys_available(ngx_quic_keys_t *keys, - enum ssl_encryption_level_t level) + enum ssl_encryption_level_t level, ngx_uint_t is_write) { - return keys->secrets[level].client.key.len != 0; + if (is_write == 0) { + return keys->secrets[level].client.key.len != 0; + } + + return keys->secrets[level].server.key.len != 0; } @@ -683,6 +687,7 @@ ngx_quic_keys_discard(ngx_quic_keys_t *keys, enum ssl_encryption_level_t level) { keys->secrets[level].client.key.len = 0; + keys->secrets[level].server.key.len = 0; } diff --git a/src/event/quic/ngx_event_quic_protection.h b/src/event/quic/ngx_event_quic_protection.h index 2d3006776ee..be76714225c 100644 --- a/src/event/quic/ngx_event_quic_protection.h +++ b/src/event/quic/ngx_event_quic_protection.h @@ -95,7 +95,7 @@ ngx_int_t ngx_quic_keys_set_encryption_secret(ngx_log_t *log, enum ssl_encryption_level_t level, const SSL_CIPHER *cipher, const uint8_t *secret, size_t secret_len); ngx_uint_t ngx_quic_keys_available(ngx_quic_keys_t *keys, - enum ssl_encryption_level_t level); + enum ssl_encryption_level_t level, ngx_uint_t is_write); void ngx_quic_keys_discard(ngx_quic_keys_t *keys, enum ssl_encryption_level_t level); void ngx_quic_keys_switch(ngx_connection_t *c, ngx_quic_keys_t *keys); diff --git a/src/event/quic/ngx_event_quic_ssl.c b/src/event/quic/ngx_event_quic_ssl.c index c719a1dd4ae..7872783f898 100644 --- a/src/event/quic/ngx_event_quic_ssl.c +++ b/src/event/quic/ngx_event_quic_ssl.c @@ -434,7 +434,7 @@ ngx_quic_crypto_input(ngx_connection_t *c, ngx_chain_t *data, } if (n <= 0 || SSL_in_init(ssl_conn)) { - if (ngx_quic_keys_available(qc->keys, ssl_encryption_early_data) + if (ngx_quic_keys_available(qc->keys, ssl_encryption_early_data, 0) && qc->client_tp_done) { if (ngx_quic_init_streams(c) != NGX_OK) { From fffd2823ba8bddcdfdffca47990c73cd4298208d Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Fri, 20 Oct 2023 18:05:07 +0400 Subject: [PATCH 053/279] QUIC: added safety belt to prevent using discarded keys. In addition to triggering alert, it ensures that such packets won't be sent. With the previous change that marks server keys as discarded by zeroing the key lengh, it is now an error to send packets with discarded keys. OpenSSL based stacks tolerate such behaviour because key length isn't used in packet protection, but BoringSSL will raise the UNSUPPORTED_KEY_SIZE cipher error. It won't be possible to use discarded keys with reused crypto contexts as it happens in subsequent changes. --- src/event/quic/ngx_event_quic_output.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/event/quic/ngx_event_quic_output.c b/src/event/quic/ngx_event_quic_output.c index 587671bc6a9..bd3e7e3b0e7 100644 --- a/src/event/quic/ngx_event_quic_output.c +++ b/src/event/quic/ngx_event_quic_output.c @@ -519,6 +519,21 @@ ngx_quic_output_packet(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx, qc = ngx_quic_get_connection(c); + if (!ngx_quic_keys_available(qc->keys, ctx->level, 1)) { + ngx_log_error(NGX_LOG_ALERT, c->log, 0, "quic %s write keys discarded", + ngx_quic_level_name(ctx->level)); + + while (!ngx_queue_empty(&ctx->frames)) { + q = ngx_queue_head(&ctx->frames); + ngx_queue_remove(q); + + f = ngx_queue_data(q, ngx_quic_frame_t, queue); + ngx_quic_free_frame(c, f); + } + + return 0; + } + ngx_quic_init_packet(c, ctx, &pkt, qc->path); min_payload = ngx_quic_payload_size(&pkt, min); From 8e1217c46dcd66efa1cc2ca2d2b43d85db24b416 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Fri, 20 Oct 2023 18:05:07 +0400 Subject: [PATCH 054/279] QUIC: prevented generating ACK frames with discarded keys. Previously it was possible to generate ACK frames using formally discarded protection keys, in particular, when acknowledging a client Handshake packet used to complete the TLS handshake and to discard handshake protection keys. As it happens late in packet processing, it could be possible to generate ACK frames after the keys were already discarded. ACK frames are generated from ngx_quic_ack_packet(), either using a posted push event, which envolves ngx_quic_generate_ack() as a part of the final packet assembling, or directly in ngx_quic_ack_packet(), such as when there is no room to add a new ACK range or when the received packet is out of order. The added keys availability check is used to avoid generating late ACK frames in both cases. --- src/event/quic/ngx_event_quic_ack.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/event/quic/ngx_event_quic_ack.c b/src/event/quic/ngx_event_quic_ack.c index 23c9af34829..a7fd85d20bd 100644 --- a/src/event/quic/ngx_event_quic_ack.c +++ b/src/event/quic/ngx_event_quic_ack.c @@ -907,6 +907,10 @@ ngx_quic_ack_packet(ngx_connection_t *c, ngx_quic_header_t *pkt) " nranges:%ui", pkt->pn, (int64_t) ctx->largest_range, ctx->first_range, ctx->nranges); + if (!ngx_quic_keys_available(qc->keys, ctx->level, 1)) { + return NGX_OK; + } + prev_pending = ctx->pending_ack; if (pkt->need_ack) { From 885a02696e689ea422256d58891276499c7da56f Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Fri, 20 Oct 2023 18:05:07 +0400 Subject: [PATCH 055/279] QUIC: renamed protection functions. Now these functions have names ngx_quic_crypto_XXX(): - ngx_quic_tls_open() -> ngx_quic_crypto_open() - ngx_quic_tls_seal() -> ngx_quic_crypto_seal() - ngx_quic_tls_hp() -> ngx_quic_crypto_hp() --- .../quic/ngx_event_quic_openssl_compat.c | 4 +-- src/event/quic/ngx_event_quic_protection.c | 25 ++++++++++--------- src/event/quic/ngx_event_quic_protection.h | 2 +- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/src/event/quic/ngx_event_quic_openssl_compat.c b/src/event/quic/ngx_event_quic_openssl_compat.c index cc6b95d482d..75981338bf3 100644 --- a/src/event/quic/ngx_event_quic_openssl_compat.c +++ b/src/event/quic/ngx_event_quic_openssl_compat.c @@ -578,8 +578,8 @@ ngx_quic_compat_create_record(ngx_quic_compat_record_t *rec, ngx_str_t *res) ngx_memcpy(nonce, secret->iv.data, secret->iv.len); ngx_quic_compute_nonce(nonce, sizeof(nonce), rec->number); - if (ngx_quic_tls_seal(ciphers.c, secret, &out, - nonce, &rec->payload, &ad, rec->log) + if (ngx_quic_crypto_seal(ciphers.c, secret, &out, + nonce, &rec->payload, &ad, rec->log) != NGX_OK) { return NGX_ERROR; diff --git a/src/event/quic/ngx_event_quic_protection.c b/src/event/quic/ngx_event_quic_protection.c index 9f8169988a6..54ee585983e 100644 --- a/src/event/quic/ngx_event_quic_protection.c +++ b/src/event/quic/ngx_event_quic_protection.c @@ -26,10 +26,10 @@ static ngx_int_t ngx_hkdf_extract(u_char *out_key, size_t *out_len, static uint64_t ngx_quic_parse_pn(u_char **pos, ngx_int_t len, u_char *mask, uint64_t *largest_pn); -static ngx_int_t ngx_quic_tls_open(const ngx_quic_cipher_t *cipher, +static ngx_int_t ngx_quic_crypto_open(const ngx_quic_cipher_t *cipher, ngx_quic_secret_t *s, ngx_str_t *out, u_char *nonce, ngx_str_t *in, ngx_str_t *ad, ngx_log_t *log); -static ngx_int_t ngx_quic_tls_hp(ngx_log_t *log, const EVP_CIPHER *cipher, +static ngx_int_t ngx_quic_crypto_hp(ngx_log_t *log, const EVP_CIPHER *cipher, ngx_quic_secret_t *s, u_char *out, u_char *in); static ngx_int_t ngx_quic_create_packet(ngx_quic_header_t *pkt, @@ -344,7 +344,7 @@ ngx_hkdf_extract(u_char *out_key, size_t *out_len, const EVP_MD *digest, static ngx_int_t -ngx_quic_tls_open(const ngx_quic_cipher_t *cipher, ngx_quic_secret_t *s, +ngx_quic_crypto_open(const ngx_quic_cipher_t *cipher, ngx_quic_secret_t *s, ngx_str_t *out, u_char *nonce, ngx_str_t *in, ngx_str_t *ad, ngx_log_t *log) { @@ -449,7 +449,7 @@ ngx_quic_tls_open(const ngx_quic_cipher_t *cipher, ngx_quic_secret_t *s, ngx_int_t -ngx_quic_tls_seal(const ngx_quic_cipher_t *cipher, ngx_quic_secret_t *s, +ngx_quic_crypto_seal(const ngx_quic_cipher_t *cipher, ngx_quic_secret_t *s, ngx_str_t *out, u_char *nonce, ngx_str_t *in, ngx_str_t *ad, ngx_log_t *log) { @@ -565,7 +565,7 @@ ngx_quic_tls_seal(const ngx_quic_cipher_t *cipher, ngx_quic_secret_t *s, static ngx_int_t -ngx_quic_tls_hp(ngx_log_t *log, const EVP_CIPHER *cipher, +ngx_quic_crypto_hp(ngx_log_t *log, const EVP_CIPHER *cipher, ngx_quic_secret_t *s, u_char *out, u_char *in) { int outlen; @@ -801,15 +801,15 @@ ngx_quic_create_packet(ngx_quic_header_t *pkt, ngx_str_t *res) ngx_memcpy(nonce, secret->iv.data, secret->iv.len); ngx_quic_compute_nonce(nonce, sizeof(nonce), pkt->number); - if (ngx_quic_tls_seal(ciphers.c, secret, &out, - nonce, &pkt->payload, &ad, pkt->log) + if (ngx_quic_crypto_seal(ciphers.c, secret, &out, + nonce, &pkt->payload, &ad, pkt->log) != NGX_OK) { return NGX_ERROR; } sample = &out.data[4 - pkt->num_len]; - if (ngx_quic_tls_hp(pkt->log, ciphers.hp, secret, mask, sample) + if (ngx_quic_crypto_hp(pkt->log, ciphers.hp, secret, mask, sample) != NGX_OK) { return NGX_ERROR; @@ -862,7 +862,8 @@ ngx_quic_create_retry_packet(ngx_quic_header_t *pkt, ngx_str_t *res) ngx_memcpy(secret.key.data, key, sizeof(key)); secret.iv.len = NGX_QUIC_IV_LEN; - if (ngx_quic_tls_seal(ciphers.c, &secret, &itag, nonce, &in, &ad, pkt->log) + if (ngx_quic_crypto_seal(ciphers.c, &secret, &itag, nonce, &in, &ad, + pkt->log) != NGX_OK) { return NGX_ERROR; @@ -1032,7 +1033,7 @@ ngx_quic_decrypt(ngx_quic_header_t *pkt, uint64_t *largest_pn) /* header protection */ - if (ngx_quic_tls_hp(pkt->log, ciphers.hp, secret, mask, sample) + if (ngx_quic_crypto_hp(pkt->log, ciphers.hp, secret, mask, sample) != NGX_OK) { return NGX_DECLINED; @@ -1087,8 +1088,8 @@ ngx_quic_decrypt(ngx_quic_header_t *pkt, uint64_t *largest_pn) pkt->payload.len = in.len - NGX_QUIC_TAG_LEN; pkt->payload.data = pkt->plaintext + ad.len; - rc = ngx_quic_tls_open(ciphers.c, secret, &pkt->payload, - nonce, &in, &ad, pkt->log); + rc = ngx_quic_crypto_open(ciphers.c, secret, &pkt->payload, + nonce, &in, &ad, pkt->log); if (rc != NGX_OK) { return NGX_DECLINED; } diff --git a/src/event/quic/ngx_event_quic_protection.h b/src/event/quic/ngx_event_quic_protection.h index be76714225c..3c415d6957a 100644 --- a/src/event/quic/ngx_event_quic_protection.h +++ b/src/event/quic/ngx_event_quic_protection.h @@ -105,7 +105,7 @@ ngx_int_t ngx_quic_decrypt(ngx_quic_header_t *pkt, uint64_t *largest_pn); void ngx_quic_compute_nonce(u_char *nonce, size_t len, uint64_t pn); ngx_int_t ngx_quic_ciphers(ngx_uint_t id, ngx_quic_ciphers_t *ciphers, enum ssl_encryption_level_t level); -ngx_int_t ngx_quic_tls_seal(const ngx_quic_cipher_t *cipher, +ngx_int_t ngx_quic_crypto_seal(const ngx_quic_cipher_t *cipher, ngx_quic_secret_t *s, ngx_str_t *out, u_char *nonce, ngx_str_t *in, ngx_str_t *ad, ngx_log_t *log); ngx_int_t ngx_quic_hkdf_expand(ngx_quic_hkdf_t *hkdf, const EVP_MD *digest, From 80a695add847f78b93dfb1b09aeb5562a2558d2a Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Fri, 20 Oct 2023 18:05:07 +0400 Subject: [PATCH 056/279] QUIC: reusing crypto contexts for packet protection. --- src/event/quic/ngx_event_quic.c | 3 + .../quic/ngx_event_quic_openssl_compat.c | 45 ++- src/event/quic/ngx_event_quic_output.c | 4 + src/event/quic/ngx_event_quic_protection.c | 270 +++++++++++------- src/event/quic/ngx_event_quic_protection.h | 12 +- 5 files changed, 213 insertions(+), 121 deletions(-) diff --git a/src/event/quic/ngx_event_quic.c b/src/event/quic/ngx_event_quic.c index c9cd527a153..b0cf056c1ee 100644 --- a/src/event/quic/ngx_event_quic.c +++ b/src/event/quic/ngx_event_quic.c @@ -335,6 +335,7 @@ ngx_quic_new_connection(ngx_connection_t *c, ngx_quic_conf_t *conf, qc->validated = pkt->validated; if (ngx_quic_open_sockets(c, qc, pkt) != NGX_OK) { + ngx_quic_keys_cleanup(qc->keys); return NULL; } @@ -585,6 +586,8 @@ ngx_quic_close_connection(ngx_connection_t *c, ngx_int_t rc) ngx_quic_close_sockets(c); + ngx_quic_keys_cleanup(qc->keys); + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic close completed"); /* may be tested from SSL callback during SSL shutdown */ diff --git a/src/event/quic/ngx_event_quic_openssl_compat.c b/src/event/quic/ngx_event_quic_openssl_compat.c index 75981338bf3..a6121f5a663 100644 --- a/src/event/quic/ngx_event_quic_openssl_compat.c +++ b/src/event/quic/ngx_event_quic_openssl_compat.c @@ -54,9 +54,10 @@ struct ngx_quic_compat_s { static void ngx_quic_compat_keylog_callback(const SSL *ssl, const char *line); -static ngx_int_t ngx_quic_compat_set_encryption_secret(ngx_log_t *log, +static ngx_int_t ngx_quic_compat_set_encryption_secret(ngx_connection_t *c, ngx_quic_compat_keys_t *keys, enum ssl_encryption_level_t level, const SSL_CIPHER *cipher, const uint8_t *secret, size_t secret_len); +static void ngx_quic_compat_cleanup_encryption_secret(void *data); static int ngx_quic_compat_add_transport_params_callback(SSL *ssl, unsigned int ext_type, unsigned int context, const unsigned char **out, size_t *outlen, X509 *x, size_t chainidx, int *al, void *add_arg); @@ -214,14 +215,14 @@ ngx_quic_compat_keylog_callback(const SSL *ssl, const char *line) com->method->set_read_secret((SSL *) ssl, level, cipher, secret, n); com->read_record = 0; - (void) ngx_quic_compat_set_encryption_secret(c->log, &com->keys, level, + (void) ngx_quic_compat_set_encryption_secret(c, &com->keys, level, cipher, secret, n); } } static ngx_int_t -ngx_quic_compat_set_encryption_secret(ngx_log_t *log, +ngx_quic_compat_set_encryption_secret(ngx_connection_t *c, ngx_quic_compat_keys_t *keys, enum ssl_encryption_level_t level, const SSL_CIPHER *cipher, const uint8_t *secret, size_t secret_len) { @@ -231,6 +232,7 @@ ngx_quic_compat_set_encryption_secret(ngx_log_t *log, ngx_quic_hkdf_t seq[2]; ngx_quic_secret_t *peer_secret; ngx_quic_ciphers_t ciphers; + ngx_pool_cleanup_t *cln; peer_secret = &keys->secret; @@ -239,12 +241,12 @@ ngx_quic_compat_set_encryption_secret(ngx_log_t *log, key_len = ngx_quic_ciphers(keys->cipher, &ciphers, level); if (key_len == NGX_ERROR) { - ngx_ssl_error(NGX_LOG_INFO, log, 0, "unexpected cipher"); + ngx_ssl_error(NGX_LOG_INFO, c->log, 0, "unexpected cipher"); return NGX_ERROR; } if (sizeof(peer_secret->secret.data) < secret_len) { - ngx_log_error(NGX_LOG_ALERT, log, 0, + ngx_log_error(NGX_LOG_ALERT, c->log, 0, "unexpected secret len: %uz", secret_len); return NGX_ERROR; } @@ -262,15 +264,43 @@ ngx_quic_compat_set_encryption_secret(ngx_log_t *log, ngx_quic_hkdf_set(&seq[1], "tls13 iv", &peer_secret->iv, &secret_str); for (i = 0; i < (sizeof(seq) / sizeof(seq[0])); i++) { - if (ngx_quic_hkdf_expand(&seq[i], ciphers.d, log) != NGX_OK) { + if (ngx_quic_hkdf_expand(&seq[i], ciphers.d, c->log) != NGX_OK) { return NGX_ERROR; } } + /* register cleanup handler once */ + + if (peer_secret->ctx) { + ngx_quic_crypto_cleanup(peer_secret); + + } else { + cln = ngx_pool_cleanup_add(c->pool, 0); + if (cln == NULL) { + return NGX_ERROR; + } + + cln->handler = ngx_quic_compat_cleanup_encryption_secret; + cln->data = peer_secret; + } + + if (ngx_quic_crypto_init(ciphers.c, peer_secret, 1, c->log) == NGX_ERROR) { + return NGX_ERROR; + } + return NGX_OK; } +static void +ngx_quic_compat_cleanup_encryption_secret(void *data) +{ + ngx_quic_secret_t *secret = data; + + ngx_quic_crypto_cleanup(secret); +} + + static int ngx_quic_compat_add_transport_params_callback(SSL *ssl, unsigned int ext_type, unsigned int context, const unsigned char **out, size_t *outlen, X509 *x, @@ -578,8 +608,7 @@ ngx_quic_compat_create_record(ngx_quic_compat_record_t *rec, ngx_str_t *res) ngx_memcpy(nonce, secret->iv.data, secret->iv.len); ngx_quic_compute_nonce(nonce, sizeof(nonce), rec->number); - if (ngx_quic_crypto_seal(ciphers.c, secret, &out, - nonce, &rec->payload, &ad, rec->log) + if (ngx_quic_crypto_seal(secret, &out, nonce, &rec->payload, &ad, rec->log) != NGX_OK) { return NGX_ERROR; diff --git a/src/event/quic/ngx_event_quic_output.c b/src/event/quic/ngx_event_quic_output.c index bd3e7e3b0e7..dd528a7febf 100644 --- a/src/event/quic/ngx_event_quic_output.c +++ b/src/event/quic/ngx_event_quic_output.c @@ -941,13 +941,17 @@ ngx_quic_send_early_cc(ngx_connection_t *c, ngx_quic_header_t *inpkt, res.data = dst; if (ngx_quic_encrypt(&pkt, &res) != NGX_OK) { + ngx_quic_keys_cleanup(pkt.keys); return NGX_ERROR; } if (ngx_quic_send(c, res.data, res.len, c->sockaddr, c->socklen) < 0) { + ngx_quic_keys_cleanup(pkt.keys); return NGX_ERROR; } + ngx_quic_keys_cleanup(pkt.keys); + return NGX_DONE; } diff --git a/src/event/quic/ngx_event_quic_protection.c b/src/event/quic/ngx_event_quic_protection.c index 54ee585983e..98b961a0e4e 100644 --- a/src/event/quic/ngx_event_quic_protection.c +++ b/src/event/quic/ngx_event_quic_protection.c @@ -26,9 +26,8 @@ static ngx_int_t ngx_hkdf_extract(u_char *out_key, size_t *out_len, static uint64_t ngx_quic_parse_pn(u_char **pos, ngx_int_t len, u_char *mask, uint64_t *largest_pn); -static ngx_int_t ngx_quic_crypto_open(const ngx_quic_cipher_t *cipher, - ngx_quic_secret_t *s, ngx_str_t *out, u_char *nonce, ngx_str_t *in, - ngx_str_t *ad, ngx_log_t *log); +static ngx_int_t ngx_quic_crypto_open(ngx_quic_secret_t *s, ngx_str_t *out, + u_char *nonce, ngx_str_t *in, ngx_str_t *ad, ngx_log_t *log); static ngx_int_t ngx_quic_crypto_hp(ngx_log_t *log, const EVP_CIPHER *cipher, ngx_quic_secret_t *s, u_char *out, u_char *in); @@ -108,13 +107,14 @@ ngx_int_t ngx_quic_keys_set_initial_secret(ngx_quic_keys_t *keys, ngx_str_t *secret, ngx_log_t *log) { - size_t is_len; - uint8_t is[SHA256_DIGEST_LENGTH]; - ngx_str_t iss; - ngx_uint_t i; - const EVP_MD *digest; - ngx_quic_hkdf_t seq[8]; - ngx_quic_secret_t *client, *server; + size_t is_len; + uint8_t is[SHA256_DIGEST_LENGTH]; + ngx_str_t iss; + ngx_uint_t i; + const EVP_MD *digest; + ngx_quic_hkdf_t seq[8]; + ngx_quic_secret_t *client, *server; + ngx_quic_ciphers_t ciphers; static const uint8_t salt[20] = "\x38\x76\x2c\xf7\xf5\x59\x34\xb3\x4d\x17" @@ -180,7 +180,25 @@ ngx_quic_keys_set_initial_secret(ngx_quic_keys_t *keys, ngx_str_t *secret, } } + if (ngx_quic_ciphers(0, &ciphers, ssl_encryption_initial) == NGX_ERROR) { + return NGX_ERROR; + } + + if (ngx_quic_crypto_init(ciphers.c, client, 0, log) == NGX_ERROR) { + return NGX_ERROR; + } + + if (ngx_quic_crypto_init(ciphers.c, server, 1, log) == NGX_ERROR) { + goto failed; + } + return NGX_OK; + +failed: + + ngx_quic_keys_cleanup(keys); + + return NGX_ERROR; } @@ -343,9 +361,9 @@ ngx_hkdf_extract(u_char *out_key, size_t *out_len, const EVP_MD *digest, } -static ngx_int_t -ngx_quic_crypto_open(const ngx_quic_cipher_t *cipher, ngx_quic_secret_t *s, - ngx_str_t *out, u_char *nonce, ngx_str_t *in, ngx_str_t *ad, ngx_log_t *log) +ngx_int_t +ngx_quic_crypto_init(const ngx_quic_cipher_t *cipher, ngx_quic_secret_t *s, + ngx_int_t enc, ngx_log_t *log) { #ifdef OPENSSL_IS_BORINGSSL @@ -357,19 +375,7 @@ ngx_quic_crypto_open(const ngx_quic_cipher_t *cipher, ngx_quic_secret_t *s, ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_AEAD_CTX_new() failed"); return NGX_ERROR; } - - if (EVP_AEAD_CTX_open(ctx, out->data, &out->len, out->len, nonce, s->iv.len, - in->data, in->len, ad->data, ad->len) - != 1) - { - EVP_AEAD_CTX_free(ctx); - ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_AEAD_CTX_open() failed"); - return NGX_ERROR; - } - - EVP_AEAD_CTX_free(ctx); #else - int len; EVP_CIPHER_CTX *ctx; ctx = EVP_CIPHER_CTX_new(); @@ -378,17 +384,16 @@ ngx_quic_crypto_open(const ngx_quic_cipher_t *cipher, ngx_quic_secret_t *s, return NGX_ERROR; } - if (EVP_DecryptInit_ex(ctx, cipher, NULL, NULL, NULL) != 1) { + if (EVP_CipherInit_ex(ctx, cipher, NULL, NULL, NULL, enc) != 1) { EVP_CIPHER_CTX_free(ctx); - ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_DecryptInit_ex() failed"); + ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_CipherInit_ex() failed"); return NGX_ERROR; } - in->len -= NGX_QUIC_TAG_LEN; - - if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_SET_TAG, NGX_QUIC_TAG_LEN, - in->data + in->len) - == 0) + if (EVP_CIPHER_mode(cipher) == EVP_CIPH_CCM_MODE + && EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_SET_TAG, NGX_QUIC_TAG_LEN, + NULL) + == 0) { EVP_CIPHER_CTX_free(ctx); ngx_ssl_error(NGX_LOG_INFO, log, 0, @@ -405,28 +410,66 @@ ngx_quic_crypto_open(const ngx_quic_cipher_t *cipher, ngx_quic_secret_t *s, return NGX_ERROR; } - if (EVP_DecryptInit_ex(ctx, NULL, NULL, s->key.data, nonce) != 1) { + if (EVP_CipherInit_ex(ctx, NULL, NULL, s->key.data, NULL, enc) != 1) { EVP_CIPHER_CTX_free(ctx); + ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_CipherInit_ex() failed"); + return NGX_ERROR; + } +#endif + + s->ctx = ctx; + return NGX_OK; +} + + +static ngx_int_t +ngx_quic_crypto_open(ngx_quic_secret_t *s, ngx_str_t *out, u_char *nonce, + ngx_str_t *in, ngx_str_t *ad, ngx_log_t *log) +{ + ngx_quic_crypto_ctx_t *ctx; + + ctx = s->ctx; + +#ifdef OPENSSL_IS_BORINGSSL + if (EVP_AEAD_CTX_open(ctx, out->data, &out->len, out->len, nonce, s->iv.len, + in->data, in->len, ad->data, ad->len) + != 1) + { + ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_AEAD_CTX_open() failed"); + return NGX_ERROR; + } +#else + int len; + + if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, nonce) != 1) { ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_DecryptInit_ex() failed"); return NGX_ERROR; } - if (EVP_CIPHER_mode(cipher) == EVP_CIPH_CCM_MODE + in->len -= NGX_QUIC_TAG_LEN; + + if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_SET_TAG, NGX_QUIC_TAG_LEN, + in->data + in->len) + == 0) + { + ngx_ssl_error(NGX_LOG_INFO, log, 0, + "EVP_CIPHER_CTX_ctrl(EVP_CTRL_AEAD_SET_TAG) failed"); + return NGX_ERROR; + } + + if (EVP_CIPHER_mode(EVP_CIPHER_CTX_cipher(ctx)) == EVP_CIPH_CCM_MODE && EVP_DecryptUpdate(ctx, NULL, &len, NULL, in->len) != 1) { - EVP_CIPHER_CTX_free(ctx); ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_DecryptUpdate() failed"); return NGX_ERROR; } if (EVP_DecryptUpdate(ctx, NULL, &len, ad->data, ad->len) != 1) { - EVP_CIPHER_CTX_free(ctx); ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_DecryptUpdate() failed"); return NGX_ERROR; } if (EVP_DecryptUpdate(ctx, out->data, &len, in->data, in->len) != 1) { - EVP_CIPHER_CTX_free(ctx); ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_DecryptUpdate() failed"); return NGX_ERROR; } @@ -434,14 +477,11 @@ ngx_quic_crypto_open(const ngx_quic_cipher_t *cipher, ngx_quic_secret_t *s, out->len = len; if (EVP_DecryptFinal_ex(ctx, out->data + out->len, &len) <= 0) { - EVP_CIPHER_CTX_free(ctx); ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_DecryptFinal_ex failed"); return NGX_ERROR; } out->len += len; - - EVP_CIPHER_CTX_free(ctx); #endif return NGX_OK; @@ -449,88 +489,42 @@ ngx_quic_crypto_open(const ngx_quic_cipher_t *cipher, ngx_quic_secret_t *s, ngx_int_t -ngx_quic_crypto_seal(const ngx_quic_cipher_t *cipher, ngx_quic_secret_t *s, - ngx_str_t *out, u_char *nonce, ngx_str_t *in, ngx_str_t *ad, ngx_log_t *log) +ngx_quic_crypto_seal(ngx_quic_secret_t *s, ngx_str_t *out, u_char *nonce, + ngx_str_t *in, ngx_str_t *ad, ngx_log_t *log) { + ngx_quic_crypto_ctx_t *ctx; -#ifdef OPENSSL_IS_BORINGSSL - EVP_AEAD_CTX *ctx; - - ctx = EVP_AEAD_CTX_new(cipher, s->key.data, s->key.len, - EVP_AEAD_DEFAULT_TAG_LENGTH); - if (ctx == NULL) { - ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_AEAD_CTX_new() failed"); - return NGX_ERROR; - } + ctx = s->ctx; +#ifdef OPENSSL_IS_BORINGSSL if (EVP_AEAD_CTX_seal(ctx, out->data, &out->len, out->len, nonce, s->iv.len, in->data, in->len, ad->data, ad->len) != 1) { - EVP_AEAD_CTX_free(ctx); ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_AEAD_CTX_seal() failed"); return NGX_ERROR; } - - EVP_AEAD_CTX_free(ctx); #else - int len; - EVP_CIPHER_CTX *ctx; + int len; - ctx = EVP_CIPHER_CTX_new(); - if (ctx == NULL) { - ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_CIPHER_CTX_new() failed"); - return NGX_ERROR; - } - - if (EVP_EncryptInit_ex(ctx, cipher, NULL, NULL, NULL) != 1) { - EVP_CIPHER_CTX_free(ctx); + if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, nonce) != 1) { ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_EncryptInit_ex() failed"); return NGX_ERROR; } - if (EVP_CIPHER_mode(cipher) == EVP_CIPH_CCM_MODE - && EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_SET_TAG, NGX_QUIC_TAG_LEN, - NULL) - == 0) - { - EVP_CIPHER_CTX_free(ctx); - ngx_ssl_error(NGX_LOG_INFO, log, 0, - "EVP_CIPHER_CTX_ctrl(EVP_CTRL_AEAD_SET_TAG) failed"); - return NGX_ERROR; - } - - if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_SET_IVLEN, s->iv.len, NULL) - == 0) - { - EVP_CIPHER_CTX_free(ctx); - ngx_ssl_error(NGX_LOG_INFO, log, 0, - "EVP_CIPHER_CTX_ctrl(EVP_CTRL_AEAD_SET_IVLEN) failed"); - return NGX_ERROR; - } - - if (EVP_EncryptInit_ex(ctx, NULL, NULL, s->key.data, nonce) != 1) { - EVP_CIPHER_CTX_free(ctx); - ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_EncryptInit_ex() failed"); - return NGX_ERROR; - } - - if (EVP_CIPHER_mode(cipher) == EVP_CIPH_CCM_MODE + if (EVP_CIPHER_mode(EVP_CIPHER_CTX_cipher(ctx)) == EVP_CIPH_CCM_MODE && EVP_EncryptUpdate(ctx, NULL, &len, NULL, in->len) != 1) { - EVP_CIPHER_CTX_free(ctx); ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_EncryptUpdate() failed"); return NGX_ERROR; } if (EVP_EncryptUpdate(ctx, NULL, &len, ad->data, ad->len) != 1) { - EVP_CIPHER_CTX_free(ctx); ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_EncryptUpdate() failed"); return NGX_ERROR; } if (EVP_EncryptUpdate(ctx, out->data, &len, in->data, in->len) != 1) { - EVP_CIPHER_CTX_free(ctx); ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_EncryptUpdate() failed"); return NGX_ERROR; } @@ -538,7 +532,6 @@ ngx_quic_crypto_seal(const ngx_quic_cipher_t *cipher, ngx_quic_secret_t *s, out->len = len; if (EVP_EncryptFinal_ex(ctx, out->data + out->len, &len) <= 0) { - EVP_CIPHER_CTX_free(ctx); ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_EncryptFinal_ex failed"); return NGX_ERROR; } @@ -549,21 +542,32 @@ ngx_quic_crypto_seal(const ngx_quic_cipher_t *cipher, ngx_quic_secret_t *s, out->data + out->len) == 0) { - EVP_CIPHER_CTX_free(ctx); ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_CIPHER_CTX_ctrl(EVP_CTRL_AEAD_GET_TAG) failed"); return NGX_ERROR; } out->len += NGX_QUIC_TAG_LEN; - - EVP_CIPHER_CTX_free(ctx); #endif return NGX_OK; } +void +ngx_quic_crypto_cleanup(ngx_quic_secret_t *s) +{ + if (s->ctx) { +#ifdef OPENSSL_IS_BORINGSSL + EVP_AEAD_CTX_free(s->ctx); +#else + EVP_CIPHER_CTX_free(s->ctx); +#endif + s->ctx = NULL; + } +} + + static ngx_int_t ngx_quic_crypto_hp(ngx_log_t *log, const EVP_CIPHER *cipher, ngx_quic_secret_t *s, u_char *out, u_char *in) @@ -666,6 +670,12 @@ ngx_quic_keys_set_encryption_secret(ngx_log_t *log, ngx_uint_t is_write, } } + if (ngx_quic_crypto_init(ciphers.c, peer_secret, is_write, log) + == NGX_ERROR) + { + return NGX_ERROR; + } + return NGX_OK; } @@ -675,10 +685,10 @@ ngx_quic_keys_available(ngx_quic_keys_t *keys, enum ssl_encryption_level_t level, ngx_uint_t is_write) { if (is_write == 0) { - return keys->secrets[level].client.key.len != 0; + return keys->secrets[level].client.ctx != NULL; } - return keys->secrets[level].server.key.len != 0; + return keys->secrets[level].server.ctx != NULL; } @@ -686,8 +696,13 @@ void ngx_quic_keys_discard(ngx_quic_keys_t *keys, enum ssl_encryption_level_t level) { - keys->secrets[level].client.key.len = 0; - keys->secrets[level].server.key.len = 0; + ngx_quic_secret_t *client, *server; + + client = &keys->secrets[level].client; + server = &keys->secrets[level].server; + + ngx_quic_crypto_cleanup(client); + ngx_quic_crypto_cleanup(server); } @@ -699,6 +714,9 @@ ngx_quic_keys_switch(ngx_connection_t *c, ngx_quic_keys_t *keys) current = &keys->secrets[ssl_encryption_application]; next = &keys->next_key; + ngx_quic_crypto_cleanup(¤t->client); + ngx_quic_crypto_cleanup(¤t->server); + tmp = *current; *current = *next; *next = tmp; @@ -762,6 +780,16 @@ ngx_quic_keys_update(ngx_event_t *ev) } } + if (ngx_quic_crypto_init(ciphers.c, &next->client, 0, c->log) == NGX_ERROR) + { + goto failed; + } + + if (ngx_quic_crypto_init(ciphers.c, &next->server, 1, c->log) == NGX_ERROR) + { + goto failed; + } + return; failed: @@ -770,6 +798,23 @@ ngx_quic_keys_update(ngx_event_t *ev) } +void +ngx_quic_keys_cleanup(ngx_quic_keys_t *keys) +{ + ngx_uint_t i; + ngx_quic_secrets_t *next; + + for (i = 0; i < NGX_QUIC_ENCRYPTION_LAST; i++) { + ngx_quic_keys_discard(keys, i); + } + + next = &keys->next_key; + + ngx_quic_crypto_cleanup(&next->client); + ngx_quic_crypto_cleanup(&next->server); +} + + static ngx_int_t ngx_quic_create_packet(ngx_quic_header_t *pkt, ngx_str_t *res) { @@ -801,8 +846,7 @@ ngx_quic_create_packet(ngx_quic_header_t *pkt, ngx_str_t *res) ngx_memcpy(nonce, secret->iv.data, secret->iv.len); ngx_quic_compute_nonce(nonce, sizeof(nonce), pkt->number); - if (ngx_quic_crypto_seal(ciphers.c, secret, &out, - nonce, &pkt->payload, &ad, pkt->log) + if (ngx_quic_crypto_seal(secret, &out, nonce, &pkt->payload, &ad, pkt->log) != NGX_OK) { return NGX_ERROR; @@ -862,13 +906,19 @@ ngx_quic_create_retry_packet(ngx_quic_header_t *pkt, ngx_str_t *res) ngx_memcpy(secret.key.data, key, sizeof(key)); secret.iv.len = NGX_QUIC_IV_LEN; - if (ngx_quic_crypto_seal(ciphers.c, &secret, &itag, nonce, &in, &ad, - pkt->log) + if (ngx_quic_crypto_init(ciphers.c, &secret, 1, pkt->log) == NGX_ERROR) { + return NGX_ERROR; + } + + if (ngx_quic_crypto_seal(&secret, &itag, nonce, &in, &ad, pkt->log) != NGX_OK) { + ngx_quic_crypto_cleanup(&secret); return NGX_ERROR; } + ngx_quic_crypto_cleanup(&secret); + res->len = itag.data + itag.len - start; res->data = start; @@ -999,7 +1049,7 @@ ngx_quic_decrypt(ngx_quic_header_t *pkt, uint64_t *largest_pn) u_char *p, *sample; size_t len; uint64_t pn, lpn; - ngx_int_t pnl, rc; + ngx_int_t pnl; ngx_str_t in, ad; ngx_uint_t key_phase; ngx_quic_secret_t *secret; @@ -1088,9 +1138,9 @@ ngx_quic_decrypt(ngx_quic_header_t *pkt, uint64_t *largest_pn) pkt->payload.len = in.len - NGX_QUIC_TAG_LEN; pkt->payload.data = pkt->plaintext + ad.len; - rc = ngx_quic_crypto_open(ciphers.c, secret, &pkt->payload, - nonce, &in, &ad, pkt->log); - if (rc != NGX_OK) { + if (ngx_quic_crypto_open(secret, &pkt->payload, nonce, &in, &ad, pkt->log) + != NGX_OK) + { return NGX_DECLINED; } diff --git a/src/event/quic/ngx_event_quic_protection.h b/src/event/quic/ngx_event_quic_protection.h index 3c415d6957a..99c8c9aa112 100644 --- a/src/event/quic/ngx_event_quic_protection.h +++ b/src/event/quic/ngx_event_quic_protection.h @@ -26,8 +26,10 @@ #ifdef OPENSSL_IS_BORINGSSL #define ngx_quic_cipher_t EVP_AEAD +#define ngx_quic_crypto_ctx_t EVP_AEAD_CTX #else #define ngx_quic_cipher_t EVP_CIPHER +#define ngx_quic_crypto_ctx_t EVP_CIPHER_CTX #endif @@ -48,6 +50,7 @@ typedef struct { ngx_quic_md_t key; ngx_quic_iv_t iv; ngx_quic_md_t hp; + ngx_quic_crypto_ctx_t *ctx; } ngx_quic_secret_t; @@ -100,14 +103,17 @@ void ngx_quic_keys_discard(ngx_quic_keys_t *keys, enum ssl_encryption_level_t level); void ngx_quic_keys_switch(ngx_connection_t *c, ngx_quic_keys_t *keys); void ngx_quic_keys_update(ngx_event_t *ev); +void ngx_quic_keys_cleanup(ngx_quic_keys_t *keys); ngx_int_t ngx_quic_encrypt(ngx_quic_header_t *pkt, ngx_str_t *res); ngx_int_t ngx_quic_decrypt(ngx_quic_header_t *pkt, uint64_t *largest_pn); void ngx_quic_compute_nonce(u_char *nonce, size_t len, uint64_t pn); ngx_int_t ngx_quic_ciphers(ngx_uint_t id, ngx_quic_ciphers_t *ciphers, enum ssl_encryption_level_t level); -ngx_int_t ngx_quic_crypto_seal(const ngx_quic_cipher_t *cipher, - ngx_quic_secret_t *s, ngx_str_t *out, u_char *nonce, ngx_str_t *in, - ngx_str_t *ad, ngx_log_t *log); +ngx_int_t ngx_quic_crypto_init(const ngx_quic_cipher_t *cipher, + ngx_quic_secret_t *s, ngx_int_t enc, ngx_log_t *log); +ngx_int_t ngx_quic_crypto_seal(ngx_quic_secret_t *s, ngx_str_t *out, + u_char *nonce, ngx_str_t *in, ngx_str_t *ad, ngx_log_t *log); +void ngx_quic_crypto_cleanup(ngx_quic_secret_t *s); ngx_int_t ngx_quic_hkdf_expand(ngx_quic_hkdf_t *hkdf, const EVP_MD *digest, ngx_log_t *log); From 52d50714eb0ffd46796b52526da273db7b4a3883 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Fri, 20 Oct 2023 18:05:07 +0400 Subject: [PATCH 057/279] QUIC: common code for crypto open and seal operations. --- src/event/quic/ngx_event_quic_protection.c | 139 ++++++++++----------- 1 file changed, 63 insertions(+), 76 deletions(-) diff --git a/src/event/quic/ngx_event_quic_protection.c b/src/event/quic/ngx_event_quic_protection.c index 98b961a0e4e..a3edc7d5eba 100644 --- a/src/event/quic/ngx_event_quic_protection.c +++ b/src/event/quic/ngx_event_quic_protection.c @@ -28,6 +28,10 @@ static uint64_t ngx_quic_parse_pn(u_char **pos, ngx_int_t len, u_char *mask, static ngx_int_t ngx_quic_crypto_open(ngx_quic_secret_t *s, ngx_str_t *out, u_char *nonce, ngx_str_t *in, ngx_str_t *ad, ngx_log_t *log); +#ifndef OPENSSL_IS_BORINGSSL +static ngx_int_t ngx_quic_crypto_common(ngx_quic_secret_t *s, ngx_str_t *out, + u_char *nonce, ngx_str_t *in, ngx_str_t *ad, ngx_log_t *log); +#endif static ngx_int_t ngx_quic_crypto_hp(ngx_log_t *log, const EVP_CIPHER *cipher, ngx_quic_secret_t *s, u_char *out, u_char *in); @@ -426,133 +430,116 @@ static ngx_int_t ngx_quic_crypto_open(ngx_quic_secret_t *s, ngx_str_t *out, u_char *nonce, ngx_str_t *in, ngx_str_t *ad, ngx_log_t *log) { - ngx_quic_crypto_ctx_t *ctx; - - ctx = s->ctx; - #ifdef OPENSSL_IS_BORINGSSL - if (EVP_AEAD_CTX_open(ctx, out->data, &out->len, out->len, nonce, s->iv.len, - in->data, in->len, ad->data, ad->len) + if (EVP_AEAD_CTX_open(s->ctx, out->data, &out->len, out->len, nonce, + s->iv.len, in->data, in->len, ad->data, ad->len) != 1) { ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_AEAD_CTX_open() failed"); return NGX_ERROR; } -#else - int len; - if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, nonce) != 1) { - ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_DecryptInit_ex() failed"); - return NGX_ERROR; - } - - in->len -= NGX_QUIC_TAG_LEN; + return NGX_OK; +#else + return ngx_quic_crypto_common(s, out, nonce, in, ad, log); +#endif +} - if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_SET_TAG, NGX_QUIC_TAG_LEN, - in->data + in->len) - == 0) - { - ngx_ssl_error(NGX_LOG_INFO, log, 0, - "EVP_CIPHER_CTX_ctrl(EVP_CTRL_AEAD_SET_TAG) failed"); - return NGX_ERROR; - } - if (EVP_CIPHER_mode(EVP_CIPHER_CTX_cipher(ctx)) == EVP_CIPH_CCM_MODE - && EVP_DecryptUpdate(ctx, NULL, &len, NULL, in->len) != 1) +ngx_int_t +ngx_quic_crypto_seal(ngx_quic_secret_t *s, ngx_str_t *out, u_char *nonce, + ngx_str_t *in, ngx_str_t *ad, ngx_log_t *log) +{ +#ifdef OPENSSL_IS_BORINGSSL + if (EVP_AEAD_CTX_seal(s->ctx, out->data, &out->len, out->len, nonce, + s->iv.len, in->data, in->len, ad->data, ad->len) + != 1) { - ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_DecryptUpdate() failed"); - return NGX_ERROR; - } - - if (EVP_DecryptUpdate(ctx, NULL, &len, ad->data, ad->len) != 1) { - ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_DecryptUpdate() failed"); - return NGX_ERROR; - } - - if (EVP_DecryptUpdate(ctx, out->data, &len, in->data, in->len) != 1) { - ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_DecryptUpdate() failed"); - return NGX_ERROR; - } - - out->len = len; - - if (EVP_DecryptFinal_ex(ctx, out->data + out->len, &len) <= 0) { - ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_DecryptFinal_ex failed"); + ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_AEAD_CTX_seal() failed"); return NGX_ERROR; } - out->len += len; -#endif - return NGX_OK; +#else + return ngx_quic_crypto_common(s, out, nonce, in, ad, log); +#endif } -ngx_int_t -ngx_quic_crypto_seal(ngx_quic_secret_t *s, ngx_str_t *out, u_char *nonce, +#ifndef OPENSSL_IS_BORINGSSL + +static ngx_int_t +ngx_quic_crypto_common(ngx_quic_secret_t *s, ngx_str_t *out, u_char *nonce, ngx_str_t *in, ngx_str_t *ad, ngx_log_t *log) { + int len, enc; ngx_quic_crypto_ctx_t *ctx; ctx = s->ctx; + enc = EVP_CIPHER_CTX_encrypting(ctx); -#ifdef OPENSSL_IS_BORINGSSL - if (EVP_AEAD_CTX_seal(ctx, out->data, &out->len, out->len, nonce, s->iv.len, - in->data, in->len, ad->data, ad->len) - != 1) - { - ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_AEAD_CTX_seal() failed"); + if (EVP_CipherInit_ex(ctx, NULL, NULL, NULL, nonce, enc) != 1) { + ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_CipherInit_ex() failed"); return NGX_ERROR; } -#else - int len; - if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, nonce) != 1) { - ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_EncryptInit_ex() failed"); - return NGX_ERROR; + if (enc == 0) { + in->len -= NGX_QUIC_TAG_LEN; + + if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_SET_TAG, NGX_QUIC_TAG_LEN, + in->data + in->len) + == 0) + { + ngx_ssl_error(NGX_LOG_INFO, log, 0, + "EVP_CIPHER_CTX_ctrl(EVP_CTRL_AEAD_SET_TAG) failed"); + return NGX_ERROR; + } } if (EVP_CIPHER_mode(EVP_CIPHER_CTX_cipher(ctx)) == EVP_CIPH_CCM_MODE - && EVP_EncryptUpdate(ctx, NULL, &len, NULL, in->len) != 1) + && EVP_CipherUpdate(ctx, NULL, &len, NULL, in->len) != 1) { - ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_EncryptUpdate() failed"); + ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_CipherUpdate() failed"); return NGX_ERROR; } - if (EVP_EncryptUpdate(ctx, NULL, &len, ad->data, ad->len) != 1) { - ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_EncryptUpdate() failed"); + if (EVP_CipherUpdate(ctx, NULL, &len, ad->data, ad->len) != 1) { + ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_CipherUpdate() failed"); return NGX_ERROR; } - if (EVP_EncryptUpdate(ctx, out->data, &len, in->data, in->len) != 1) { - ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_EncryptUpdate() failed"); + if (EVP_CipherUpdate(ctx, out->data, &len, in->data, in->len) != 1) { + ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_CipherUpdate() failed"); return NGX_ERROR; } out->len = len; - if (EVP_EncryptFinal_ex(ctx, out->data + out->len, &len) <= 0) { - ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_EncryptFinal_ex failed"); + if (EVP_CipherFinal_ex(ctx, out->data + out->len, &len) <= 0) { + ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_CipherFinal_ex failed"); return NGX_ERROR; } out->len += len; - if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_GET_TAG, NGX_QUIC_TAG_LEN, - out->data + out->len) - == 0) - { - ngx_ssl_error(NGX_LOG_INFO, log, 0, - "EVP_CIPHER_CTX_ctrl(EVP_CTRL_AEAD_GET_TAG) failed"); - return NGX_ERROR; - } + if (enc == 1) { + if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_GET_TAG, NGX_QUIC_TAG_LEN, + out->data + out->len) + == 0) + { + ngx_ssl_error(NGX_LOG_INFO, log, 0, + "EVP_CIPHER_CTX_ctrl(EVP_CTRL_AEAD_GET_TAG) failed"); + return NGX_ERROR; + } - out->len += NGX_QUIC_TAG_LEN; -#endif + out->len += NGX_QUIC_TAG_LEN; + } return NGX_OK; } +#endif + void ngx_quic_crypto_cleanup(ngx_quic_secret_t *s) From 4f60ee789e5746491c2acf6a834fbb41a6dbd982 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Fri, 20 Oct 2023 18:05:07 +0400 Subject: [PATCH 058/279] QUIC: reusing crypto contexts for header protection. --- src/event/quic/ngx_event_quic_protection.c | 102 +++++++++++++++------ src/event/quic/ngx_event_quic_protection.h | 1 + 2 files changed, 75 insertions(+), 28 deletions(-) diff --git a/src/event/quic/ngx_event_quic_protection.c b/src/event/quic/ngx_event_quic_protection.c index a3edc7d5eba..c3f2365b340 100644 --- a/src/event/quic/ngx_event_quic_protection.c +++ b/src/event/quic/ngx_event_quic_protection.c @@ -32,8 +32,12 @@ static ngx_int_t ngx_quic_crypto_open(ngx_quic_secret_t *s, ngx_str_t *out, static ngx_int_t ngx_quic_crypto_common(ngx_quic_secret_t *s, ngx_str_t *out, u_char *nonce, ngx_str_t *in, ngx_str_t *ad, ngx_log_t *log); #endif -static ngx_int_t ngx_quic_crypto_hp(ngx_log_t *log, const EVP_CIPHER *cipher, - ngx_quic_secret_t *s, u_char *out, u_char *in); + +static ngx_int_t ngx_quic_crypto_hp_init(const EVP_CIPHER *cipher, + ngx_quic_secret_t *s, ngx_log_t *log); +static ngx_int_t ngx_quic_crypto_hp(ngx_quic_secret_t *s, + u_char *out, u_char *in, ngx_log_t *log); +static void ngx_quic_crypto_hp_cleanup(ngx_quic_secret_t *s); static ngx_int_t ngx_quic_create_packet(ngx_quic_header_t *pkt, ngx_str_t *res); @@ -196,6 +200,14 @@ ngx_quic_keys_set_initial_secret(ngx_quic_keys_t *keys, ngx_str_t *secret, goto failed; } + if (ngx_quic_crypto_hp_init(ciphers.hp, client, log) == NGX_ERROR) { + goto failed; + } + + if (ngx_quic_crypto_hp_init(ciphers.hp, server, log) == NGX_ERROR) { + goto failed; + } + return NGX_OK; failed: @@ -556,53 +568,82 @@ ngx_quic_crypto_cleanup(ngx_quic_secret_t *s) static ngx_int_t -ngx_quic_crypto_hp(ngx_log_t *log, const EVP_CIPHER *cipher, - ngx_quic_secret_t *s, u_char *out, u_char *in) +ngx_quic_crypto_hp_init(const EVP_CIPHER *cipher, ngx_quic_secret_t *s, + ngx_log_t *log) { - int outlen; EVP_CIPHER_CTX *ctx; - u_char zero[NGX_QUIC_HP_LEN] = {0}; #ifdef OPENSSL_IS_BORINGSSL - uint32_t cnt; - - ngx_memcpy(&cnt, in, sizeof(uint32_t)); - - if (cipher == (const EVP_CIPHER *) EVP_aead_chacha20_poly1305()) { - CRYPTO_chacha_20(out, zero, NGX_QUIC_HP_LEN, s->hp.data, &in[4], cnt); + if (cipher == (EVP_CIPHER *) EVP_aead_chacha20_poly1305()) { + /* no EVP interface */ + s->hp_ctx = NULL; return NGX_OK; } #endif ctx = EVP_CIPHER_CTX_new(); if (ctx == NULL) { + ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_CIPHER_CTX_new() failed"); return NGX_ERROR; } - if (EVP_EncryptInit_ex(ctx, cipher, NULL, s->hp.data, in) != 1) { + if (EVP_EncryptInit_ex(ctx, cipher, NULL, s->hp.data, NULL) != 1) { + EVP_CIPHER_CTX_free(ctx); ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_EncryptInit_ex() failed"); - goto failed; + return NGX_ERROR; + } + + s->hp_ctx = ctx; + return NGX_OK; +} + + +static ngx_int_t +ngx_quic_crypto_hp(ngx_quic_secret_t *s, u_char *out, u_char *in, + ngx_log_t *log) +{ + int outlen; + EVP_CIPHER_CTX *ctx; + u_char zero[NGX_QUIC_HP_LEN] = {0}; + + ctx = s->hp_ctx; + +#ifdef OPENSSL_IS_BORINGSSL + uint32_t cnt; + + if (ctx == NULL) { + ngx_memcpy(&cnt, in, sizeof(uint32_t)); + CRYPTO_chacha_20(out, zero, NGX_QUIC_HP_LEN, s->hp.data, &in[4], cnt); + return NGX_OK; + } +#endif + + if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, in) != 1) { + ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_EncryptInit_ex() failed"); + return NGX_ERROR; } if (!EVP_EncryptUpdate(ctx, out, &outlen, zero, NGX_QUIC_HP_LEN)) { ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_EncryptUpdate() failed"); - goto failed; + return NGX_ERROR; } if (!EVP_EncryptFinal_ex(ctx, out + NGX_QUIC_HP_LEN, &outlen)) { ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_EncryptFinal_Ex() failed"); - goto failed; + return NGX_ERROR; } - EVP_CIPHER_CTX_free(ctx); - return NGX_OK; +} -failed: - - EVP_CIPHER_CTX_free(ctx); - return NGX_ERROR; +static void +ngx_quic_crypto_hp_cleanup(ngx_quic_secret_t *s) +{ + if (s->hp_ctx) { + EVP_CIPHER_CTX_free(s->hp_ctx); + s->hp_ctx = NULL; + } } @@ -663,6 +704,10 @@ ngx_quic_keys_set_encryption_secret(ngx_log_t *log, ngx_uint_t is_write, return NGX_ERROR; } + if (ngx_quic_crypto_hp_init(ciphers.hp, peer_secret, log) == NGX_ERROR) { + return NGX_ERROR; + } + return NGX_OK; } @@ -690,6 +735,9 @@ ngx_quic_keys_discard(ngx_quic_keys_t *keys, ngx_quic_crypto_cleanup(client); ngx_quic_crypto_cleanup(server); + + ngx_quic_crypto_hp_cleanup(client); + ngx_quic_crypto_hp_cleanup(server); } @@ -742,11 +790,13 @@ ngx_quic_keys_update(ngx_event_t *ev) next->client.key.len = current->client.key.len; next->client.iv.len = NGX_QUIC_IV_LEN; next->client.hp = current->client.hp; + next->client.hp_ctx = current->client.hp_ctx; next->server.secret.len = current->server.secret.len; next->server.key.len = current->server.key.len; next->server.iv.len = NGX_QUIC_IV_LEN; next->server.hp = current->server.hp; + next->server.hp_ctx = current->server.hp_ctx; ngx_quic_hkdf_set(&seq[0], "tls13 quic ku", &next->client.secret, ¤t->client.secret); @@ -840,9 +890,7 @@ ngx_quic_create_packet(ngx_quic_header_t *pkt, ngx_str_t *res) } sample = &out.data[4 - pkt->num_len]; - if (ngx_quic_crypto_hp(pkt->log, ciphers.hp, secret, mask, sample) - != NGX_OK) - { + if (ngx_quic_crypto_hp(secret, mask, sample, pkt->log) != NGX_OK) { return NGX_ERROR; } @@ -1070,9 +1118,7 @@ ngx_quic_decrypt(ngx_quic_header_t *pkt, uint64_t *largest_pn) /* header protection */ - if (ngx_quic_crypto_hp(pkt->log, ciphers.hp, secret, mask, sample) - != NGX_OK) - { + if (ngx_quic_crypto_hp(secret, mask, sample, pkt->log) != NGX_OK) { return NGX_DECLINED; } diff --git a/src/event/quic/ngx_event_quic_protection.h b/src/event/quic/ngx_event_quic_protection.h index 99c8c9aa112..d0aeec14693 100644 --- a/src/event/quic/ngx_event_quic_protection.h +++ b/src/event/quic/ngx_event_quic_protection.h @@ -51,6 +51,7 @@ typedef struct { ngx_quic_iv_t iv; ngx_quic_md_t hp; ngx_quic_crypto_ctx_t *ctx; + EVP_CIPHER_CTX *hp_ctx; } ngx_quic_secret_t; From d15f8f2c85ebd01d3da0428ac05bf0b1b36e102b Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Fri, 20 Oct 2023 18:05:07 +0400 Subject: [PATCH 059/279] QUIC: cleaned up now unused ngx_quic_ciphers() calls. --- .../quic/ngx_event_quic_openssl_compat.c | 12 ++---- src/event/quic/ngx_event_quic_protection.c | 38 +++++++------------ 2 files changed, 16 insertions(+), 34 deletions(-) diff --git a/src/event/quic/ngx_event_quic_openssl_compat.c b/src/event/quic/ngx_event_quic_openssl_compat.c index a6121f5a663..44f4cb1b38f 100644 --- a/src/event/quic/ngx_event_quic_openssl_compat.c +++ b/src/event/quic/ngx_event_quic_openssl_compat.c @@ -582,10 +582,9 @@ ngx_quic_compat_create_header(ngx_quic_compat_record_t *rec, u_char *out, static ngx_int_t ngx_quic_compat_create_record(ngx_quic_compat_record_t *rec, ngx_str_t *res) { - ngx_str_t ad, out; - ngx_quic_secret_t *secret; - ngx_quic_ciphers_t ciphers; - u_char nonce[NGX_QUIC_IV_LEN]; + ngx_str_t ad, out; + ngx_quic_secret_t *secret; + u_char nonce[NGX_QUIC_IV_LEN]; ad.data = res->data; ad.len = ngx_quic_compat_create_header(rec, ad.data, 0); @@ -598,11 +597,6 @@ ngx_quic_compat_create_record(ngx_quic_compat_record_t *rec, ngx_str_t *res) "quic compat ad len:%uz %xV", ad.len, &ad); #endif - if (ngx_quic_ciphers(rec->keys->cipher, &ciphers, rec->level) == NGX_ERROR) - { - return NGX_ERROR; - } - secret = &rec->keys->secret; ngx_memcpy(nonce, secret->iv.data, secret->iv.len); diff --git a/src/event/quic/ngx_event_quic_protection.c b/src/event/quic/ngx_event_quic_protection.c index c3f2365b340..d7ef8d23f11 100644 --- a/src/event/quic/ngx_event_quic_protection.c +++ b/src/event/quic/ngx_event_quic_protection.c @@ -855,12 +855,11 @@ ngx_quic_keys_cleanup(ngx_quic_keys_t *keys) static ngx_int_t ngx_quic_create_packet(ngx_quic_header_t *pkt, ngx_str_t *res) { - u_char *pnp, *sample; - ngx_str_t ad, out; - ngx_uint_t i; - ngx_quic_secret_t *secret; - ngx_quic_ciphers_t ciphers; - u_char nonce[NGX_QUIC_IV_LEN], mask[NGX_QUIC_HP_LEN]; + u_char *pnp, *sample; + ngx_str_t ad, out; + ngx_uint_t i; + ngx_quic_secret_t *secret; + u_char nonce[NGX_QUIC_IV_LEN], mask[NGX_QUIC_HP_LEN]; ad.data = res->data; ad.len = ngx_quic_create_header(pkt, ad.data, &pnp); @@ -873,11 +872,6 @@ ngx_quic_create_packet(ngx_quic_header_t *pkt, ngx_str_t *res) "quic ad len:%uz %xV", ad.len, &ad); #endif - if (ngx_quic_ciphers(pkt->keys->cipher, &ciphers, pkt->level) == NGX_ERROR) - { - return NGX_ERROR; - } - secret = &pkt->keys->secrets[pkt->level].server; ngx_memcpy(nonce, secret->iv.data, secret->iv.len); @@ -1081,20 +1075,14 @@ ngx_quic_encrypt(ngx_quic_header_t *pkt, ngx_str_t *res) ngx_int_t ngx_quic_decrypt(ngx_quic_header_t *pkt, uint64_t *largest_pn) { - u_char *p, *sample; - size_t len; - uint64_t pn, lpn; - ngx_int_t pnl; - ngx_str_t in, ad; - ngx_uint_t key_phase; - ngx_quic_secret_t *secret; - ngx_quic_ciphers_t ciphers; - uint8_t nonce[NGX_QUIC_IV_LEN], mask[NGX_QUIC_HP_LEN]; - - if (ngx_quic_ciphers(pkt->keys->cipher, &ciphers, pkt->level) == NGX_ERROR) - { - return NGX_ERROR; - } + u_char *p, *sample; + size_t len; + uint64_t pn, lpn; + ngx_int_t pnl; + ngx_str_t in, ad; + ngx_uint_t key_phase; + ngx_quic_secret_t *secret; + uint8_t nonce[NGX_QUIC_IV_LEN], mask[NGX_QUIC_HP_LEN]; secret = &pkt->keys->secrets[pkt->level].client; From 01bd8cacebafedf4540bc97d040ddc644aaa5418 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Fri, 20 Oct 2023 18:05:07 +0400 Subject: [PATCH 060/279] QUIC: simplified ngx_quic_ciphers() API. After conversion to reusable crypto ctx, now there's enough caller context to remove the "level" argument from ngx_quic_ciphers(). --- .../quic/ngx_event_quic_openssl_compat.c | 2 +- src/event/quic/ngx_event_quic_protection.c | 19 +++++++------------ src/event/quic/ngx_event_quic_protection.h | 3 +-- 3 files changed, 9 insertions(+), 15 deletions(-) diff --git a/src/event/quic/ngx_event_quic_openssl_compat.c b/src/event/quic/ngx_event_quic_openssl_compat.c index 44f4cb1b38f..1ece946c696 100644 --- a/src/event/quic/ngx_event_quic_openssl_compat.c +++ b/src/event/quic/ngx_event_quic_openssl_compat.c @@ -238,7 +238,7 @@ ngx_quic_compat_set_encryption_secret(ngx_connection_t *c, keys->cipher = SSL_CIPHER_get_id(cipher); - key_len = ngx_quic_ciphers(keys->cipher, &ciphers, level); + key_len = ngx_quic_ciphers(keys->cipher, &ciphers); if (key_len == NGX_ERROR) { ngx_ssl_error(NGX_LOG_INFO, c->log, 0, "unexpected cipher"); diff --git a/src/event/quic/ngx_event_quic_protection.c b/src/event/quic/ngx_event_quic_protection.c index d7ef8d23f11..038f8c0df53 100644 --- a/src/event/quic/ngx_event_quic_protection.c +++ b/src/event/quic/ngx_event_quic_protection.c @@ -15,6 +15,8 @@ #define NGX_QUIC_AES_128_KEY_LEN 16 +#define NGX_QUIC_INITIAL_CIPHER TLS1_3_CK_AES_128_GCM_SHA256 + static ngx_int_t ngx_hkdf_expand(u_char *out_key, size_t out_len, const EVP_MD *digest, const u_char *prk, size_t prk_len, @@ -46,15 +48,10 @@ static ngx_int_t ngx_quic_create_retry_packet(ngx_quic_header_t *pkt, ngx_int_t -ngx_quic_ciphers(ngx_uint_t id, ngx_quic_ciphers_t *ciphers, - enum ssl_encryption_level_t level) +ngx_quic_ciphers(ngx_uint_t id, ngx_quic_ciphers_t *ciphers) { ngx_int_t len; - if (level == ssl_encryption_initial) { - id = TLS1_3_CK_AES_128_GCM_SHA256; - } - switch (id) { case TLS1_3_CK_AES_128_GCM_SHA256: @@ -188,7 +185,7 @@ ngx_quic_keys_set_initial_secret(ngx_quic_keys_t *keys, ngx_str_t *secret, } } - if (ngx_quic_ciphers(0, &ciphers, ssl_encryption_initial) == NGX_ERROR) { + if (ngx_quic_ciphers(NGX_QUIC_INITIAL_CIPHER, &ciphers) == NGX_ERROR) { return NGX_ERROR; } @@ -664,7 +661,7 @@ ngx_quic_keys_set_encryption_secret(ngx_log_t *log, ngx_uint_t is_write, keys->cipher = SSL_CIPHER_get_id(cipher); - key_len = ngx_quic_ciphers(keys->cipher, &ciphers, level); + key_len = ngx_quic_ciphers(keys->cipher, &ciphers); if (key_len == NGX_ERROR) { ngx_ssl_error(NGX_LOG_INFO, log, 0, "unexpected cipher"); @@ -780,9 +777,7 @@ ngx_quic_keys_update(ngx_event_t *ev) c->log->action = "updating keys"; - if (ngx_quic_ciphers(keys->cipher, &ciphers, ssl_encryption_application) - == NGX_ERROR) - { + if (ngx_quic_ciphers(keys->cipher, &ciphers) == NGX_ERROR) { goto failed; } @@ -927,7 +922,7 @@ ngx_quic_create_retry_packet(ngx_quic_header_t *pkt, ngx_str_t *res) "quic retry itag len:%uz %xV", ad.len, &ad); #endif - if (ngx_quic_ciphers(0, &ciphers, pkt->level) == NGX_ERROR) { + if (ngx_quic_ciphers(NGX_QUIC_INITIAL_CIPHER, &ciphers) == NGX_ERROR) { return NGX_ERROR; } diff --git a/src/event/quic/ngx_event_quic_protection.h b/src/event/quic/ngx_event_quic_protection.h index d0aeec14693..a77c257f019 100644 --- a/src/event/quic/ngx_event_quic_protection.h +++ b/src/event/quic/ngx_event_quic_protection.h @@ -108,8 +108,7 @@ void ngx_quic_keys_cleanup(ngx_quic_keys_t *keys); ngx_int_t ngx_quic_encrypt(ngx_quic_header_t *pkt, ngx_str_t *res); ngx_int_t ngx_quic_decrypt(ngx_quic_header_t *pkt, uint64_t *largest_pn); void ngx_quic_compute_nonce(u_char *nonce, size_t len, uint64_t pn); -ngx_int_t ngx_quic_ciphers(ngx_uint_t id, ngx_quic_ciphers_t *ciphers, - enum ssl_encryption_level_t level); +ngx_int_t ngx_quic_ciphers(ngx_uint_t id, ngx_quic_ciphers_t *ciphers); ngx_int_t ngx_quic_crypto_init(const ngx_quic_cipher_t *cipher, ngx_quic_secret_t *s, ngx_int_t enc, ngx_log_t *log); ngx_int_t ngx_quic_crypto_seal(ngx_quic_secret_t *s, ngx_str_t *out, From b94f1fbee375eaceebf0cf0645ff83ae1966e55a Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Fri, 20 Oct 2023 18:05:07 +0400 Subject: [PATCH 061/279] QUIC: removed key field from ngx_quic_secret_t. It is made local as it is only needed now when creating crypto context. BoringSSL lacks EVP interface for ChaCha20, providing instead a function for one-shot encryption, thus hp is still preserved. Based on a patch by Roman Arutyunyan. --- .../quic/ngx_event_quic_openssl_compat.c | 10 ++- src/event/quic/ngx_event_quic_protection.c | 63 ++++++++++++------- src/event/quic/ngx_event_quic_protection.h | 3 +- 3 files changed, 47 insertions(+), 29 deletions(-) diff --git a/src/event/quic/ngx_event_quic_openssl_compat.c b/src/event/quic/ngx_event_quic_openssl_compat.c index 1ece946c696..cf4a2c895ad 100644 --- a/src/event/quic/ngx_event_quic_openssl_compat.c +++ b/src/event/quic/ngx_event_quic_openssl_compat.c @@ -229,6 +229,7 @@ ngx_quic_compat_set_encryption_secret(ngx_connection_t *c, ngx_int_t key_len; ngx_str_t secret_str; ngx_uint_t i; + ngx_quic_md_t key; ngx_quic_hkdf_t seq[2]; ngx_quic_secret_t *peer_secret; ngx_quic_ciphers_t ciphers; @@ -254,13 +255,14 @@ ngx_quic_compat_set_encryption_secret(ngx_connection_t *c, peer_secret->secret.len = secret_len; ngx_memcpy(peer_secret->secret.data, secret, secret_len); - peer_secret->key.len = key_len; + key.len = key_len; + peer_secret->iv.len = NGX_QUIC_IV_LEN; secret_str.len = secret_len; secret_str.data = (u_char *) secret; - ngx_quic_hkdf_set(&seq[0], "tls13 key", &peer_secret->key, &secret_str); + ngx_quic_hkdf_set(&seq[0], "tls13 key", &key, &secret_str); ngx_quic_hkdf_set(&seq[1], "tls13 iv", &peer_secret->iv, &secret_str); for (i = 0; i < (sizeof(seq) / sizeof(seq[0])); i++) { @@ -284,7 +286,9 @@ ngx_quic_compat_set_encryption_secret(ngx_connection_t *c, cln->data = peer_secret; } - if (ngx_quic_crypto_init(ciphers.c, peer_secret, 1, c->log) == NGX_ERROR) { + if (ngx_quic_crypto_init(ciphers.c, peer_secret, &key, 1, c->log) + == NGX_ERROR) + { return NGX_ERROR; } diff --git a/src/event/quic/ngx_event_quic_protection.c b/src/event/quic/ngx_event_quic_protection.c index 038f8c0df53..c67c4b08ac7 100644 --- a/src/event/quic/ngx_event_quic_protection.c +++ b/src/event/quic/ngx_event_quic_protection.c @@ -117,6 +117,7 @@ ngx_quic_keys_set_initial_secret(ngx_quic_keys_t *keys, ngx_str_t *secret, ngx_str_t iss; ngx_uint_t i; const EVP_MD *digest; + ngx_quic_md_t client_key, server_key; ngx_quic_hkdf_t seq[8]; ngx_quic_secret_t *client, *server; ngx_quic_ciphers_t ciphers; @@ -160,8 +161,8 @@ ngx_quic_keys_set_initial_secret(ngx_quic_keys_t *keys, ngx_str_t *secret, client->secret.len = SHA256_DIGEST_LENGTH; server->secret.len = SHA256_DIGEST_LENGTH; - client->key.len = NGX_QUIC_AES_128_KEY_LEN; - server->key.len = NGX_QUIC_AES_128_KEY_LEN; + client_key.len = NGX_QUIC_AES_128_KEY_LEN; + server_key.len = NGX_QUIC_AES_128_KEY_LEN; client->hp.len = NGX_QUIC_AES_128_KEY_LEN; server->hp.len = NGX_QUIC_AES_128_KEY_LEN; @@ -171,11 +172,11 @@ ngx_quic_keys_set_initial_secret(ngx_quic_keys_t *keys, ngx_str_t *secret, /* labels per RFC 9001, 5.1. Packet Protection Keys */ ngx_quic_hkdf_set(&seq[0], "tls13 client in", &client->secret, &iss); - ngx_quic_hkdf_set(&seq[1], "tls13 quic key", &client->key, &client->secret); + ngx_quic_hkdf_set(&seq[1], "tls13 quic key", &client_key, &client->secret); ngx_quic_hkdf_set(&seq[2], "tls13 quic iv", &client->iv, &client->secret); ngx_quic_hkdf_set(&seq[3], "tls13 quic hp", &client->hp, &client->secret); ngx_quic_hkdf_set(&seq[4], "tls13 server in", &server->secret, &iss); - ngx_quic_hkdf_set(&seq[5], "tls13 quic key", &server->key, &server->secret); + ngx_quic_hkdf_set(&seq[5], "tls13 quic key", &server_key, &server->secret); ngx_quic_hkdf_set(&seq[6], "tls13 quic iv", &server->iv, &server->secret); ngx_quic_hkdf_set(&seq[7], "tls13 quic hp", &server->hp, &server->secret); @@ -189,11 +190,15 @@ ngx_quic_keys_set_initial_secret(ngx_quic_keys_t *keys, ngx_str_t *secret, return NGX_ERROR; } - if (ngx_quic_crypto_init(ciphers.c, client, 0, log) == NGX_ERROR) { + if (ngx_quic_crypto_init(ciphers.c, client, &client_key, 0, log) + == NGX_ERROR) + { return NGX_ERROR; } - if (ngx_quic_crypto_init(ciphers.c, server, 1, log) == NGX_ERROR) { + if (ngx_quic_crypto_init(ciphers.c, server, &server_key, 1, log) + == NGX_ERROR) + { goto failed; } @@ -376,13 +381,13 @@ ngx_hkdf_extract(u_char *out_key, size_t *out_len, const EVP_MD *digest, ngx_int_t ngx_quic_crypto_init(const ngx_quic_cipher_t *cipher, ngx_quic_secret_t *s, - ngx_int_t enc, ngx_log_t *log) + ngx_quic_md_t *key, ngx_int_t enc, ngx_log_t *log) { #ifdef OPENSSL_IS_BORINGSSL EVP_AEAD_CTX *ctx; - ctx = EVP_AEAD_CTX_new(cipher, s->key.data, s->key.len, + ctx = EVP_AEAD_CTX_new(cipher, key->data, key->len, EVP_AEAD_DEFAULT_TAG_LENGTH); if (ctx == NULL) { ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_AEAD_CTX_new() failed"); @@ -423,7 +428,7 @@ ngx_quic_crypto_init(const ngx_quic_cipher_t *cipher, ngx_quic_secret_t *s, return NGX_ERROR; } - if (EVP_CipherInit_ex(ctx, NULL, NULL, s->key.data, NULL, enc) != 1) { + if (EVP_CipherInit_ex(ctx, NULL, NULL, key->data, NULL, enc) != 1) { EVP_CIPHER_CTX_free(ctx); ngx_ssl_error(NGX_LOG_INFO, log, 0, "EVP_CipherInit_ex() failed"); return NGX_ERROR; @@ -652,6 +657,7 @@ ngx_quic_keys_set_encryption_secret(ngx_log_t *log, ngx_uint_t is_write, ngx_int_t key_len; ngx_str_t secret_str; ngx_uint_t i; + ngx_quic_md_t key; ngx_quic_hkdf_t seq[3]; ngx_quic_secret_t *peer_secret; ngx_quic_ciphers_t ciphers; @@ -677,15 +683,14 @@ ngx_quic_keys_set_encryption_secret(ngx_log_t *log, ngx_uint_t is_write, peer_secret->secret.len = secret_len; ngx_memcpy(peer_secret->secret.data, secret, secret_len); - peer_secret->key.len = key_len; + key.len = key_len; peer_secret->iv.len = NGX_QUIC_IV_LEN; peer_secret->hp.len = key_len; secret_str.len = secret_len; secret_str.data = (u_char *) secret; - ngx_quic_hkdf_set(&seq[0], "tls13 quic key", - &peer_secret->key, &secret_str); + ngx_quic_hkdf_set(&seq[0], "tls13 quic key", &key, &secret_str); ngx_quic_hkdf_set(&seq[1], "tls13 quic iv", &peer_secret->iv, &secret_str); ngx_quic_hkdf_set(&seq[2], "tls13 quic hp", &peer_secret->hp, &secret_str); @@ -695,7 +700,7 @@ ngx_quic_keys_set_encryption_secret(ngx_log_t *log, ngx_uint_t is_write, } } - if (ngx_quic_crypto_init(ciphers.c, peer_secret, is_write, log) + if (ngx_quic_crypto_init(ciphers.c, peer_secret, &key, is_write, log) == NGX_ERROR) { return NGX_ERROR; @@ -758,7 +763,9 @@ ngx_quic_keys_switch(ngx_connection_t *c, ngx_quic_keys_t *keys) void ngx_quic_keys_update(ngx_event_t *ev) { + ngx_int_t key_len; ngx_uint_t i; + ngx_quic_md_t client_key, server_key; ngx_quic_hkdf_t seq[6]; ngx_quic_keys_t *keys; ngx_connection_t *c; @@ -777,18 +784,21 @@ ngx_quic_keys_update(ngx_event_t *ev) c->log->action = "updating keys"; - if (ngx_quic_ciphers(keys->cipher, &ciphers) == NGX_ERROR) { + key_len = ngx_quic_ciphers(keys->cipher, &ciphers); + + if (key_len == NGX_ERROR) { goto failed; } + client_key.len = key_len; + server_key.len = key_len; + next->client.secret.len = current->client.secret.len; - next->client.key.len = current->client.key.len; next->client.iv.len = NGX_QUIC_IV_LEN; next->client.hp = current->client.hp; next->client.hp_ctx = current->client.hp_ctx; next->server.secret.len = current->server.secret.len; - next->server.key.len = current->server.key.len; next->server.iv.len = NGX_QUIC_IV_LEN; next->server.hp = current->server.hp; next->server.hp_ctx = current->server.hp_ctx; @@ -796,13 +806,13 @@ ngx_quic_keys_update(ngx_event_t *ev) ngx_quic_hkdf_set(&seq[0], "tls13 quic ku", &next->client.secret, ¤t->client.secret); ngx_quic_hkdf_set(&seq[1], "tls13 quic key", - &next->client.key, &next->client.secret); + &client_key, &next->client.secret); ngx_quic_hkdf_set(&seq[2], "tls13 quic iv", &next->client.iv, &next->client.secret); ngx_quic_hkdf_set(&seq[3], "tls13 quic ku", &next->server.secret, ¤t->server.secret); ngx_quic_hkdf_set(&seq[4], "tls13 quic key", - &next->server.key, &next->server.secret); + &server_key, &next->server.secret); ngx_quic_hkdf_set(&seq[5], "tls13 quic iv", &next->server.iv, &next->server.secret); @@ -812,12 +822,14 @@ ngx_quic_keys_update(ngx_event_t *ev) } } - if (ngx_quic_crypto_init(ciphers.c, &next->client, 0, c->log) == NGX_ERROR) + if (ngx_quic_crypto_init(ciphers.c, &next->client, &client_key, 0, c->log) + == NGX_ERROR) { goto failed; } - if (ngx_quic_crypto_init(ciphers.c, &next->server, 1, c->log) == NGX_ERROR) + if (ngx_quic_crypto_init(ciphers.c, &next->server, &server_key, 1, c->log) + == NGX_ERROR) { goto failed; } @@ -901,11 +913,12 @@ ngx_quic_create_retry_packet(ngx_quic_header_t *pkt, ngx_str_t *res) { u_char *start; ngx_str_t ad, itag; + ngx_quic_md_t key; ngx_quic_secret_t secret; ngx_quic_ciphers_t ciphers; /* 5.8. Retry Packet Integrity */ - static u_char key[16] = + static u_char key_data[16] = "\xbe\x0c\x69\x0b\x9f\x66\x57\x5a\x1d\x76\x6b\x54\xe3\x68\xc8\x4e"; static u_char nonce[NGX_QUIC_IV_LEN] = "\x46\x15\x99\xd3\x5d\x63\x2b\xf2\x23\x98\x25\xbb"; @@ -926,11 +939,13 @@ ngx_quic_create_retry_packet(ngx_quic_header_t *pkt, ngx_str_t *res) return NGX_ERROR; } - secret.key.len = sizeof(key); - ngx_memcpy(secret.key.data, key, sizeof(key)); + key.len = sizeof(key_data); + ngx_memcpy(key.data, key_data, sizeof(key_data)); secret.iv.len = NGX_QUIC_IV_LEN; - if (ngx_quic_crypto_init(ciphers.c, &secret, 1, pkt->log) == NGX_ERROR) { + if (ngx_quic_crypto_init(ciphers.c, &secret, &key, 1, pkt->log) + == NGX_ERROR) + { return NGX_ERROR; } diff --git a/src/event/quic/ngx_event_quic_protection.h b/src/event/quic/ngx_event_quic_protection.h index a77c257f019..34cfee61be2 100644 --- a/src/event/quic/ngx_event_quic_protection.h +++ b/src/event/quic/ngx_event_quic_protection.h @@ -47,7 +47,6 @@ typedef struct { typedef struct { ngx_quic_md_t secret; - ngx_quic_md_t key; ngx_quic_iv_t iv; ngx_quic_md_t hp; ngx_quic_crypto_ctx_t *ctx; @@ -110,7 +109,7 @@ ngx_int_t ngx_quic_decrypt(ngx_quic_header_t *pkt, uint64_t *largest_pn); void ngx_quic_compute_nonce(u_char *nonce, size_t len, uint64_t pn); ngx_int_t ngx_quic_ciphers(ngx_uint_t id, ngx_quic_ciphers_t *ciphers); ngx_int_t ngx_quic_crypto_init(const ngx_quic_cipher_t *cipher, - ngx_quic_secret_t *s, ngx_int_t enc, ngx_log_t *log); + ngx_quic_secret_t *s, ngx_quic_md_t *key, ngx_int_t enc, ngx_log_t *log); ngx_int_t ngx_quic_crypto_seal(ngx_quic_secret_t *s, ngx_str_t *out, u_char *nonce, ngx_str_t *in, ngx_str_t *ad, ngx_log_t *log); void ngx_quic_crypto_cleanup(ngx_quic_secret_t *s); From 31620d1a89edaf110feda05ac91d68ccb532afba Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Fri, 20 Oct 2023 18:05:07 +0400 Subject: [PATCH 062/279] QUIC: explicitly zero out unused keying material. --- src/event/quic/ngx_event_quic_openssl_compat.c | 13 ++++--------- src/event/quic/ngx_event_quic_protection.c | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/src/event/quic/ngx_event_quic_openssl_compat.c b/src/event/quic/ngx_event_quic_openssl_compat.c index cf4a2c895ad..c7412e82be3 100644 --- a/src/event/quic/ngx_event_quic_openssl_compat.c +++ b/src/event/quic/ngx_event_quic_openssl_compat.c @@ -218,6 +218,8 @@ ngx_quic_compat_keylog_callback(const SSL *ssl, const char *line) (void) ngx_quic_compat_set_encryption_secret(c, &com->keys, level, cipher, secret, n); } + + ngx_explicit_memzero(secret, n); } @@ -246,15 +248,6 @@ ngx_quic_compat_set_encryption_secret(ngx_connection_t *c, return NGX_ERROR; } - if (sizeof(peer_secret->secret.data) < secret_len) { - ngx_log_error(NGX_LOG_ALERT, c->log, 0, - "unexpected secret len: %uz", secret_len); - return NGX_ERROR; - } - - peer_secret->secret.len = secret_len; - ngx_memcpy(peer_secret->secret.data, secret, secret_len); - key.len = key_len; peer_secret->iv.len = NGX_QUIC_IV_LEN; @@ -292,6 +285,8 @@ ngx_quic_compat_set_encryption_secret(ngx_connection_t *c, return NGX_ERROR; } + ngx_explicit_memzero(key.data, key.len); + return NGX_OK; } diff --git a/src/event/quic/ngx_event_quic_protection.c b/src/event/quic/ngx_event_quic_protection.c index c67c4b08ac7..88e6954cffb 100644 --- a/src/event/quic/ngx_event_quic_protection.c +++ b/src/event/quic/ngx_event_quic_protection.c @@ -710,6 +710,8 @@ ngx_quic_keys_set_encryption_secret(ngx_log_t *log, ngx_uint_t is_write, return NGX_ERROR; } + ngx_explicit_memzero(key.data, key.len); + return NGX_OK; } @@ -740,6 +742,9 @@ ngx_quic_keys_discard(ngx_quic_keys_t *keys, ngx_quic_crypto_hp_cleanup(client); ngx_quic_crypto_hp_cleanup(server); + + ngx_explicit_memzero(client->secret.data, client->secret.len); + ngx_explicit_memzero(server->secret.data, server->secret.len); } @@ -834,6 +839,14 @@ ngx_quic_keys_update(ngx_event_t *ev) goto failed; } + ngx_explicit_memzero(current->client.secret.data, + current->client.secret.len); + ngx_explicit_memzero(current->server.secret.data, + current->server.secret.len); + + ngx_explicit_memzero(client_key.data, client_key.len); + ngx_explicit_memzero(server_key.data, server_key.len); + return; failed: @@ -856,6 +869,11 @@ ngx_quic_keys_cleanup(ngx_quic_keys_t *keys) ngx_quic_crypto_cleanup(&next->client); ngx_quic_crypto_cleanup(&next->server); + + ngx_explicit_memzero(next->client.secret.data, + next->client.secret.len); + ngx_explicit_memzero(next->server.secret.data, + next->server.secret.len); } From b19bc2e0fa60100ee8170acf161bc9b8f01cce26 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Sat, 21 Oct 2023 18:48:24 +0400 Subject: [PATCH 063/279] HTTP/2: fixed buffer management with HTTP/2 auto-detection. As part of normal HTTP/2 processing, incomplete frames are saved in the control state using a fixed size memcpy of NGX_HTTP_V2_STATE_BUFFER_SIZE. For this matter, two state buffers are reserved in the HTTP/2 recv buffer. As part of HTTP/2 auto-detection on plain TCP connections, initial data is first read into a buffer specified by the client_header_buffer_size directive that doesn't have state reservation. Previously, this made it possible to over-read the buffer as part of saving the state. The fix is to read the available buffer size rather than a fixed size. Although memcpy of a fixed size can produce a better optimized code, handling of incomplete frames isn't a common execution path, so it was sacrificed for the sake of simplicity of the fix. --- src/http/v2/ngx_http_v2.c | 8 +++----- src/http/v2/ngx_http_v2_module.c | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/http/v2/ngx_http_v2.c b/src/http/v2/ngx_http_v2.c index 410a8be24eb..0f5bd3de8d0 100644 --- a/src/http/v2/ngx_http_v2.c +++ b/src/http/v2/ngx_http_v2.c @@ -386,13 +386,11 @@ ngx_http_v2_read_handler(ngx_event_t *rev) h2mcf = ngx_http_get_module_main_conf(h2c->http_connection->conf_ctx, ngx_http_v2_module); - available = h2mcf->recv_buffer_size - 2 * NGX_HTTP_V2_STATE_BUFFER_SIZE; + available = h2mcf->recv_buffer_size - NGX_HTTP_V2_STATE_BUFFER_SIZE; do { p = h2mcf->recv_buffer; - - ngx_memcpy(p, h2c->state.buffer, NGX_HTTP_V2_STATE_BUFFER_SIZE); - end = p + h2c->state.buffer_used; + end = ngx_cpymem(p, h2c->state.buffer, h2c->state.buffer_used); n = c->recv(c, end, available); @@ -2592,7 +2590,7 @@ ngx_http_v2_state_save(ngx_http_v2_connection_t *h2c, u_char *pos, u_char *end, return ngx_http_v2_connection_error(h2c, NGX_HTTP_V2_INTERNAL_ERROR); } - ngx_memcpy(h2c->state.buffer, pos, NGX_HTTP_V2_STATE_BUFFER_SIZE); + ngx_memcpy(h2c->state.buffer, pos, size); h2c->state.buffer_used = size; h2c->state.handler = handler; diff --git a/src/http/v2/ngx_http_v2_module.c b/src/http/v2/ngx_http_v2_module.c index 62af9a54357..d64488c20b1 100644 --- a/src/http/v2/ngx_http_v2_module.c +++ b/src/http/v2/ngx_http_v2_module.c @@ -388,7 +388,7 @@ ngx_http_v2_recv_buffer_size(ngx_conf_t *cf, void *post, void *data) { size_t *sp = data; - if (*sp <= 2 * NGX_HTTP_V2_STATE_BUFFER_SIZE) { + if (*sp <= NGX_HTTP_V2_STATE_BUFFER_SIZE) { return "value is too small"; } From 80a620a2f3f51e4dcc9555f65c3d445760cc5cdc Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Mon, 23 Oct 2023 21:50:26 +0300 Subject: [PATCH 064/279] Updated OpenSSL and zlib used for win32 builds. --- misc/GNUmakefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misc/GNUmakefile b/misc/GNUmakefile index 0853b40c2cb..067dd7481d9 100644 --- a/misc/GNUmakefile +++ b/misc/GNUmakefile @@ -6,8 +6,8 @@ TEMP = tmp CC = cl OBJS = objs.msvc8 -OPENSSL = openssl-3.0.10 -ZLIB = zlib-1.2.13 +OPENSSL = openssl-3.0.11 +ZLIB = zlib-1.3 PCRE = pcre2-10.39 From b8fb83b8d2e7ca03d43176b767f1fc657f1c1ee2 Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Tue, 24 Oct 2023 16:46:46 +0300 Subject: [PATCH 065/279] nginx-1.25.3-RELEASE --- docs/xml/nginx/changes.xml | 75 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/docs/xml/nginx/changes.xml b/docs/xml/nginx/changes.xml index 4e9919d2fb9..d2d2caccb65 100644 --- a/docs/xml/nginx/changes.xml +++ b/docs/xml/nginx/changes.xml @@ -5,6 +5,81 @@ + + + + +улучшено детектирование некорректного поведения клиентов +при использовании HTTP/2. + + +improved detection of misbehaving clients +when using HTTP/2. + + + + + +уменьшение времени запуска +при использовании большого количества location'ов.
+Спасибо Yusuke Nojima. + + +startup speedup +when using a large number of locations.
+Thanks to Yusuke Nojima. + + + + + +при использовании HTTP/2 без SSL +в рабочем процессе мог произойти segmentation fault; +ошибка появилась в 1.25.1. + + +a segmentation fault might occur in a worker process +when using HTTP/2 without SSL; +the bug had appeared in 1.25.1. + + + + + +строка "Status" в заголовке ответа бэкенда с пустой поясняющей фразой +обрабатывалась некорректно. + + +the "Status" backend response header line with an empty reason phrase +was handled incorrectly. + + + + + +утечки памяти во время переконфигурации +при использовании библиотеки PCRE2.
+Спасибо ZhenZhong Wu. + + +memory leak during reconfiguration +when using the PCRE2 library.
+Thanks to ZhenZhong Wu. + + + + + +Исправления и улучшения в HTTP/3. + + +Bugfixes and improvements in HTTP/3. + + + + + + From eb62c7f629a4da2c83b44fd819a6eb45c5658700 Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Tue, 24 Oct 2023 16:46:47 +0300 Subject: [PATCH 066/279] release-1.25.3 tag --- .hgtags | 1 + 1 file changed, 1 insertion(+) diff --git a/.hgtags b/.hgtags index 4275d556291..58d85238431 100644 --- a/.hgtags +++ b/.hgtags @@ -475,3 +475,4 @@ ac779115ed6ee4f3039e9aea414a54e560450ee2 release-1.23.4 12dcf92b0c2c68552398f19644ce3104459807d7 release-1.25.0 f8134640e8615448205785cf00b0bc810489b495 release-1.25.1 1d839f05409d1a50d0f15a2bf36547001f99ae40 release-1.25.2 +294a3d07234f8f65d7b0e0b0e2c5b05c12c5da0a release-1.25.3 From 1f1bc17ba81eb7eaf0fbadb6bd5fd711f5194d11 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Fri, 27 Oct 2023 01:29:28 +0400 Subject: [PATCH 067/279] Version bump. --- src/core/nginx.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/nginx.h b/src/core/nginx.h index 088aa1ecd32..dc1308fd13a 100644 --- a/src/core/nginx.h +++ b/src/core/nginx.h @@ -9,8 +9,8 @@ #define _NGINX_H_INCLUDED_ -#define nginx_version 1025003 -#define NGINX_VERSION "1.25.3" +#define nginx_version 1025004 +#define NGINX_VERSION "1.25.4" #define NGINX_VER "nginx/" NGINX_VERSION #ifdef NGX_BUILD From a13ed7f5ed5bebdc0b9217ffafb75ab69f835a84 Mon Sep 17 00:00:00 2001 From: Vladimir Khomutov Date: Thu, 26 Oct 2023 23:35:09 +0300 Subject: [PATCH 068/279] QUIC: improved packet and frames debug tracing. Currently, packets generated by ngx_quic_frame_sendto() and ngx_quic_send_early_cc() are not logged, thus making it hard to read logs due to gaps appearing in packet numbers sequence. At frames level, it is handy to see immediately packet number in which they arrived or being sent. --- src/event/quic/ngx_event_quic_frames.c | 4 +-- src/event/quic/ngx_event_quic_output.c | 41 +++++++++++++++-------- src/event/quic/ngx_event_quic_transport.c | 3 ++ 3 files changed, 32 insertions(+), 16 deletions(-) diff --git a/src/event/quic/ngx_event_quic_frames.c b/src/event/quic/ngx_event_quic_frames.c index 7bcfb321126..42b7d9f41de 100644 --- a/src/event/quic/ngx_event_quic_frames.c +++ b/src/event/quic/ngx_event_quic_frames.c @@ -886,8 +886,8 @@ ngx_quic_log_frame(ngx_log_t *log, ngx_quic_frame_t *f, ngx_uint_t tx) break; } - ngx_log_debug4(NGX_LOG_DEBUG_EVENT, log, 0, "quic frame %s %s %*s", - tx ? "tx" : "rx", ngx_quic_level_name(f->level), + ngx_log_debug5(NGX_LOG_DEBUG_EVENT, log, 0, "quic frame %s %s:%uL %*s", + tx ? "tx" : "rx", ngx_quic_level_name(f->level), f->pnum, p - buf, buf); } diff --git a/src/event/quic/ngx_event_quic_output.c b/src/event/quic/ngx_event_quic_output.c index dd528a7febf..914d8192155 100644 --- a/src/event/quic/ngx_event_quic_output.c +++ b/src/event/quic/ngx_event_quic_output.c @@ -35,6 +35,15 @@ #define NGX_QUIC_SOCKET_RETRY_DELAY 10 /* ms, for NGX_AGAIN on write */ +#define ngx_quic_log_packet(log, pkt) \ + ngx_log_debug6(NGX_LOG_DEBUG_EVENT, log, 0, \ + "quic packet tx %s bytes:%ui need_ack:%d" \ + " number:%L encoded nl:%d trunc:0x%xD", \ + ngx_quic_level_name((pkt)->level), (pkt)->payload.len, \ + (pkt)->need_ack, (pkt)->number, (pkt)->num_len, \ + (pkt)->trunc); + + static ngx_int_t ngx_quic_create_datagrams(ngx_connection_t *c); static void ngx_quic_commit_send(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx); static void ngx_quic_revert_send(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx, @@ -578,6 +587,11 @@ ngx_quic_output_packet(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx, pkt.need_ack = 1; } + f->pnum = ctx->pnum; + f->first = now; + f->last = now; + f->plen = 0; + ngx_quic_log_frame(c->log, f, 1); flen = ngx_quic_create_frame(p, f); @@ -588,11 +602,6 @@ ngx_quic_output_packet(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx, len += flen; p += flen; - f->pnum = ctx->pnum; - f->first = now; - f->last = now; - f->plen = 0; - nframes++; } @@ -610,11 +619,7 @@ ngx_quic_output_packet(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx, res.data = data; - ngx_log_debug6(NGX_LOG_DEBUG_EVENT, c->log, 0, - "quic packet tx %s bytes:%ui" - " need_ack:%d number:%L encoded nl:%d trunc:0x%xD", - ngx_quic_level_name(ctx->level), pkt.payload.len, - pkt.need_ack, pkt.number, pkt.num_len, pkt.trunc); + ngx_quic_log_packet(c->log, &pkt); if (ngx_quic_encrypt(&pkt, &res) != NGX_OK) { return NGX_ERROR; @@ -899,13 +904,13 @@ ngx_quic_send_early_cc(ngx_connection_t *c, ngx_quic_header_t *inpkt, frame.u.close.reason.data = (u_char *) reason; frame.u.close.reason.len = ngx_strlen(reason); + ngx_quic_log_frame(c->log, &frame, 1); + len = ngx_quic_create_frame(NULL, &frame); if (len > NGX_QUIC_MAX_UDP_PAYLOAD_SIZE) { return NGX_ERROR; } - ngx_quic_log_frame(c->log, &frame, 1); - len = ngx_quic_create_frame(src, &frame); if (len == -1) { return NGX_ERROR; @@ -940,6 +945,8 @@ ngx_quic_send_early_cc(ngx_connection_t *c, ngx_quic_header_t *inpkt, res.data = dst; + ngx_quic_log_packet(c->log, &pkt); + if (ngx_quic_encrypt(&pkt, &res) != NGX_OK) { ngx_quic_keys_cleanup(pkt.keys); return NGX_ERROR; @@ -1198,13 +1205,17 @@ ngx_quic_frame_sendto(ngx_connection_t *c, ngx_quic_frame_t *frame, pad = 4 - pkt.num_len; min_payload = ngx_max(min_payload, pad); +#if (NGX_DEBUG) + frame->pnum = pkt.number; +#endif + + ngx_quic_log_frame(c->log, frame, 1); + len = ngx_quic_create_frame(NULL, frame); if (len > NGX_QUIC_MAX_UDP_PAYLOAD_SIZE) { return NGX_ERROR; } - ngx_quic_log_frame(c->log, frame, 1); - len = ngx_quic_create_frame(src, frame); if (len == -1) { return NGX_ERROR; @@ -1220,6 +1231,8 @@ ngx_quic_frame_sendto(ngx_connection_t *c, ngx_quic_frame_t *frame, res.data = dst; + ngx_quic_log_packet(c->log, &pkt); + if (ngx_quic_encrypt(&pkt, &res) != NGX_OK) { return NGX_ERROR; } diff --git a/src/event/quic/ngx_event_quic_transport.c b/src/event/quic/ngx_event_quic_transport.c index 4e0324f4a98..19670a6b1dc 100644 --- a/src/event/quic/ngx_event_quic_transport.c +++ b/src/event/quic/ngx_event_quic_transport.c @@ -1135,6 +1135,9 @@ ngx_quic_parse_frame(ngx_quic_header_t *pkt, u_char *start, u_char *end, } f->level = pkt->level; +#if (NGX_DEBUG) + f->pnum = pkt->pn; +#endif return p - start; From 6a4eb51f5eddb0ba251d81082fa86ea2b446f34f Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Tue, 14 Nov 2023 14:50:03 +0400 Subject: [PATCH 069/279] Adjusted Huffman coding debug logging, missed in 7977:336084ff943b. Spotted by XingY Wang. --- src/http/ngx_http_huff_decode.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/http/ngx_http_huff_decode.c b/src/http/ngx_http_huff_decode.c index 14b7b78962c..a65c3c329d4 100644 --- a/src/http/ngx_http_huff_decode.c +++ b/src/http/ngx_http_huff_decode.c @@ -2657,7 +2657,7 @@ ngx_http_huff_decode(u_char *state, u_char *src, size_t len, u_char **dst, != NGX_OK) { ngx_log_debug2(NGX_LOG_DEBUG_HTTP, log, 0, - "http2 huffman decoding error at state %d: " + "http huffman decoding error at state %d: " "bad code 0x%Xd", *state, ch >> 4); return NGX_ERROR; @@ -2667,7 +2667,7 @@ ngx_http_huff_decode(u_char *state, u_char *src, size_t len, u_char **dst, != NGX_OK) { ngx_log_debug2(NGX_LOG_DEBUG_HTTP, log, 0, - "http2 huffman decoding error at state %d: " + "http huffman decoding error at state %d: " "bad code 0x%Xd", *state, ch & 0xf); return NGX_ERROR; @@ -2677,7 +2677,7 @@ ngx_http_huff_decode(u_char *state, u_char *src, size_t len, u_char **dst, if (last) { if (!ending) { ngx_log_debug1(NGX_LOG_DEBUG_HTTP, log, 0, - "http2 huffman decoding error: " + "http huffman decoding error: " "incomplete code 0x%Xd", ch); return NGX_ERROR; From f9a25736fd049075baf76a35628d2351b20f8ab8 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Tue, 14 Nov 2023 15:26:02 +0400 Subject: [PATCH 070/279] HTTP/3: added Huffman decoding error logging. --- src/http/v3/ngx_http_v3_parse.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/http/v3/ngx_http_v3_parse.c b/src/http/v3/ngx_http_v3_parse.c index c51a486c300..568816323e1 100644 --- a/src/http/v3/ngx_http_v3_parse.c +++ b/src/http/v3/ngx_http_v3_parse.c @@ -650,6 +650,8 @@ ngx_http_v3_parse_literal(ngx_connection_t *c, ngx_http_v3_parse_literal_t *st, st->length == 1, c->log) != NGX_OK) { + ngx_log_error(NGX_LOG_INFO, c->log, 0, + "client sent invalid encoded field line"); return NGX_ERROR; } From 0db94ba96a00ebfc4a3c55af8eaaf20f971a7c4c Mon Sep 17 00:00:00 2001 From: Vladimir Khomutov Date: Tue, 28 Nov 2023 12:57:14 +0300 Subject: [PATCH 071/279] HTTP: removed unused r->port_start and r->port_end. Neither r->port_start nor r->port_end were ever used. The r->port_end is set by the parser, though it was never used by the following code (and was never usable, since not copied by the ngx_http_alloc_large_header_buffer() without r->port_start set). --- src/http/ngx_http_parse.c | 3 --- src/http/ngx_http_request.c | 5 ----- src/http/ngx_http_request.h | 2 -- 3 files changed, 10 deletions(-) diff --git a/src/http/ngx_http_parse.c b/src/http/ngx_http_parse.c index d4f2dae8720..f7e50388f99 100644 --- a/src/http/ngx_http_parse.c +++ b/src/http/ngx_http_parse.c @@ -451,19 +451,16 @@ ngx_http_parse_request_line(ngx_http_request_t *r, ngx_buf_t *b) switch (ch) { case '/': - r->port_end = p; r->uri_start = p; state = sw_after_slash_in_uri; break; case '?': - r->port_end = p; r->uri_start = p; r->args_start = p + 1; r->empty_path_in_uri = 1; state = sw_uri; break; case ' ': - r->port_end = p; /* * use single "/" from request line to preserve pointers, * if request line will be copied to large client buffer diff --git a/src/http/ngx_http_request.c b/src/http/ngx_http_request.c index bd2be5eacd6..058d5028667 100644 --- a/src/http/ngx_http_request.c +++ b/src/http/ngx_http_request.c @@ -1735,11 +1735,6 @@ ngx_http_alloc_large_header_buffer(ngx_http_request_t *r, } } - if (r->port_start) { - r->port_start = new + (r->port_start - old); - r->port_end = new + (r->port_end - old); - } - if (r->uri_ext) { r->uri_ext = new + (r->uri_ext - old); } diff --git a/src/http/ngx_http_request.h b/src/http/ngx_http_request.h index cc3b7c0acba..48b052bbc56 100644 --- a/src/http/ngx_http_request.h +++ b/src/http/ngx_http_request.h @@ -597,8 +597,6 @@ struct ngx_http_request_s { u_char *schema_end; u_char *host_start; u_char *host_end; - u_char *port_start; - u_char *port_end; unsigned http_minor:16; unsigned http_major:16; From d8fa024ef1527a9aefbb52bedd70fa4449203488 Mon Sep 17 00:00:00 2001 From: Vladimir Khomutov Date: Wed, 29 Nov 2023 11:13:05 +0300 Subject: [PATCH 072/279] HTTP: uniform checks in ngx_http_alloc_large_header_buffer(). If URI is not fully parsed yet, some pointers are not set. As a result, the calculation of "new + (ptr - old)" expression is flawed. According to C11, 6.5.6 Additive operators, p.9: : When two pointers are subtracted, both shall point to elements : of the same array object, or one past the last element of the : array object Since "ptr" is not set, subtraction leads to undefined behaviour, because "ptr" and "old" are not in the same buffer (i.e. array objects). Prodded by GCC undefined behaviour sanitizer. --- src/http/ngx_http_request.c | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/src/http/ngx_http_request.c b/src/http/ngx_http_request.c index 058d5028667..21738ff4c58 100644 --- a/src/http/ngx_http_request.c +++ b/src/http/ngx_http_request.c @@ -1718,14 +1718,23 @@ ngx_http_alloc_large_header_buffer(ngx_http_request_t *r, r->request_end = new + (r->request_end - old); } - r->method_end = new + (r->method_end - old); + if (r->method_end) { + r->method_end = new + (r->method_end - old); + } + + if (r->uri_start) { + r->uri_start = new + (r->uri_start - old); + } - r->uri_start = new + (r->uri_start - old); - r->uri_end = new + (r->uri_end - old); + if (r->uri_end) { + r->uri_end = new + (r->uri_end - old); + } if (r->schema_start) { r->schema_start = new + (r->schema_start - old); - r->schema_end = new + (r->schema_end - old); + if (r->schema_end) { + r->schema_end = new + (r->schema_end - old); + } } if (r->host_start) { @@ -1749,9 +1758,18 @@ ngx_http_alloc_large_header_buffer(ngx_http_request_t *r, } else { r->header_name_start = new; - r->header_name_end = new + (r->header_name_end - old); - r->header_start = new + (r->header_start - old); - r->header_end = new + (r->header_end - old); + + if (r->header_name_end) { + r->header_name_end = new + (r->header_name_end - old); + } + + if (r->header_start) { + r->header_start = new + (r->header_start - old); + } + + if (r->header_end) { + r->header_end = new + (r->header_end - old); + } } r->header_in = b; From 0efe8db1d0c331bad799dcaa0b2a24e739f26c98 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Wed, 29 Nov 2023 18:13:25 +0400 Subject: [PATCH 073/279] QUIC: avoid partial expansion of PATH_CHALLENGE/PATH_RESPONSE. By default packets with these frames are expanded to 1200 bytes. Previously, if anti-amplification limit did not allow this expansion, it was limited to whatever size was allowed. However RFC 9000 clearly states no partial expansion should happen in both cases. Section 8.2.1. Initiating Path Validation: An endpoint MUST expand datagrams that contain a PATH_CHALLENGE frame to at least the smallest allowed maximum datagram size of 1200 bytes, unless the anti-amplification limit for the path does not permit sending a datagram of this size. Section 8.2.2. Path Validation Responses: An endpoint MUST expand datagrams that contain a PATH_RESPONSE frame to at least the smallest allowed maximum datagram size of 1200 bytes. ... However, an endpoint MUST NOT expand the datagram containing the PATH_RESPONSE if the resulting data exceeds the anti-amplification limit. --- src/event/quic/ngx_event_quic_connection.h | 3 +- src/event/quic/ngx_event_quic_migration.c | 51 ++++++++++++---------- src/event/quic/ngx_event_quic_output.c | 4 +- src/event/quic/ngx_event_quic_output.h | 2 + 4 files changed, 31 insertions(+), 29 deletions(-) diff --git a/src/event/quic/ngx_event_quic_connection.h b/src/event/quic/ngx_event_quic_connection.h index d84e3e63d33..abd0ebe6cc9 100644 --- a/src/event/quic/ngx_event_quic_connection.h +++ b/src/event/quic/ngx_event_quic_connection.h @@ -106,8 +106,7 @@ struct ngx_quic_path_s { size_t max_mtu; off_t sent; off_t received; - u_char challenge1[8]; - u_char challenge2[8]; + u_char challenge[2][8]; uint64_t seqnum; uint64_t mtu_pnum[NGX_QUIC_PATH_RETRIES]; ngx_str_t addr_text; diff --git a/src/event/quic/ngx_event_quic_migration.c b/src/event/quic/ngx_event_quic_migration.c index bcec9af1d81..2fccafa4187 100644 --- a/src/event/quic/ngx_event_quic_migration.c +++ b/src/event/quic/ngx_event_quic_migration.c @@ -36,6 +36,7 @@ ngx_int_t ngx_quic_handle_path_challenge_frame(ngx_connection_t *c, ngx_quic_header_t *pkt, ngx_quic_path_challenge_frame_t *f) { + size_t min; ngx_quic_frame_t frame, *fp; ngx_quic_connection_t *qc; @@ -57,8 +58,14 @@ ngx_quic_handle_path_challenge_frame(ngx_connection_t *c, /* * An endpoint MUST expand datagrams that contain a PATH_RESPONSE frame * to at least the smallest allowed maximum datagram size of 1200 bytes. + * ... + * However, an endpoint MUST NOT expand the datagram containing the + * PATH_RESPONSE if the resulting data exceeds the anti-amplification limit. */ - if (ngx_quic_frame_sendto(c, &frame, 1200, pkt->path) == NGX_ERROR) { + + min = (ngx_quic_path_limit(c, pkt->path, 1200) < 1200) ? 0 : 1200; + + if (ngx_quic_frame_sendto(c, &frame, min, pkt->path) == NGX_ERROR) { return NGX_ERROR; } @@ -113,8 +120,8 @@ ngx_quic_handle_path_response_frame(ngx_connection_t *c, continue; } - if (ngx_memcmp(path->challenge1, f->data, sizeof(f->data)) == 0 - || ngx_memcmp(path->challenge2, f->data, sizeof(f->data)) == 0) + if (ngx_memcmp(path->challenge[0], f->data, sizeof(f->data)) == 0 + || ngx_memcmp(path->challenge[1], f->data, sizeof(f->data)) == 0) { goto valid; } @@ -510,11 +517,7 @@ ngx_quic_validate_path(ngx_connection_t *c, ngx_quic_path_t *path) path->tries = 0; - if (RAND_bytes(path->challenge1, 8) != 1) { - return NGX_ERROR; - } - - if (RAND_bytes(path->challenge2, 8) != 1) { + if (RAND_bytes((u_char *) path->challenge, sizeof(path->challenge)) != 1) { return NGX_ERROR; } @@ -535,6 +538,8 @@ ngx_quic_validate_path(ngx_connection_t *c, ngx_quic_path_t *path) static ngx_int_t ngx_quic_send_path_challenge(ngx_connection_t *c, ngx_quic_path_t *path) { + size_t min; + ngx_uint_t n; ngx_quic_frame_t frame; ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, @@ -546,26 +551,24 @@ ngx_quic_send_path_challenge(ngx_connection_t *c, ngx_quic_path_t *path) frame.level = ssl_encryption_application; frame.type = NGX_QUIC_FT_PATH_CHALLENGE; - ngx_memcpy(frame.u.path_challenge.data, path->challenge1, 8); + for (n = 0; n < 2; n++) { - /* - * RFC 9000, 8.2.1. Initiating Path Validation - * - * An endpoint MUST expand datagrams that contain a PATH_CHALLENGE frame - * to at least the smallest allowed maximum datagram size of 1200 bytes, - * unless the anti-amplification limit for the path does not permit - * sending a datagram of this size. - */ + ngx_memcpy(frame.u.path_challenge.data, path->challenge[n], 8); - /* same applies to PATH_RESPONSE frames */ - if (ngx_quic_frame_sendto(c, &frame, 1200, path) == NGX_ERROR) { - return NGX_ERROR; - } + /* + * RFC 9000, 8.2.1. Initiating Path Validation + * + * An endpoint MUST expand datagrams that contain a PATH_CHALLENGE frame + * to at least the smallest allowed maximum datagram size of 1200 bytes, + * unless the anti-amplification limit for the path does not permit + * sending a datagram of this size. + */ - ngx_memcpy(frame.u.path_challenge.data, path->challenge2, 8); + min = (ngx_quic_path_limit(c, path, 1200) < 1200) ? 0 : 1200; - if (ngx_quic_frame_sendto(c, &frame, 1200, path) == NGX_ERROR) { - return NGX_ERROR; + if (ngx_quic_frame_sendto(c, &frame, min, path) == NGX_ERROR) { + return NGX_ERROR; + } } return NGX_OK; diff --git a/src/event/quic/ngx_event_quic_output.c b/src/event/quic/ngx_event_quic_output.c index 914d8192155..a0acdb70023 100644 --- a/src/event/quic/ngx_event_quic_output.c +++ b/src/event/quic/ngx_event_quic_output.c @@ -63,8 +63,6 @@ static ssize_t ngx_quic_send(ngx_connection_t *c, u_char *buf, size_t len, struct sockaddr *sockaddr, socklen_t socklen); static void ngx_quic_set_packet_number(ngx_quic_header_t *pkt, ngx_quic_send_ctx_t *ctx); -static size_t ngx_quic_path_limit(ngx_connection_t *c, ngx_quic_path_t *path, - size_t size); ngx_int_t @@ -1250,7 +1248,7 @@ ngx_quic_frame_sendto(ngx_connection_t *c, ngx_quic_frame_t *frame, } -static size_t +size_t ngx_quic_path_limit(ngx_connection_t *c, ngx_quic_path_t *path, size_t size) { off_t max; diff --git a/src/event/quic/ngx_event_quic_output.h b/src/event/quic/ngx_event_quic_output.h index 19f8990f46b..52d8a374f77 100644 --- a/src/event/quic/ngx_event_quic_output.h +++ b/src/event/quic/ngx_event_quic_output.h @@ -34,5 +34,7 @@ ngx_int_t ngx_quic_send_ack_range(ngx_connection_t *c, ngx_int_t ngx_quic_frame_sendto(ngx_connection_t *c, ngx_quic_frame_t *frame, size_t min, ngx_quic_path_t *path); +size_t ngx_quic_path_limit(ngx_connection_t *c, ngx_quic_path_t *path, + size_t size); #endif /* _NGX_EVENT_QUIC_OUTPUT_H_INCLUDED_ */ From 6c78bb9bb1ccbf91f5f059c13a82badea529012a Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Wed, 22 Nov 2023 14:52:21 +0400 Subject: [PATCH 074/279] QUIC: fixed anti-amplification with explicit send. Previously, when using ngx_quic_frame_sendto() to explicitly send a packet with a single frame, anti-amplification limit was not properly enforced. Even when there was no quota left for the packet, it was sent anyway, but with no padding. Now the packet is not sent at all. This function is called to send PATH_CHALLENGE/PATH_RESPONSE, PMTUD and probe packets. For all these cases packet send is retried later in case the send was not successful. --- src/event/quic/ngx_event_quic_migration.c | 6 ++++++ src/event/quic/ngx_event_quic_output.c | 24 +++++++++++++++-------- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/src/event/quic/ngx_event_quic_migration.c b/src/event/quic/ngx_event_quic_migration.c index 2fccafa4187..efb167b0ade 100644 --- a/src/event/quic/ngx_event_quic_migration.c +++ b/src/event/quic/ngx_event_quic_migration.c @@ -872,6 +872,7 @@ ngx_quic_expire_path_mtu_discovery(ngx_connection_t *c, ngx_quic_path_t *path) static ngx_int_t ngx_quic_send_path_mtu_probe(ngx_connection_t *c, ngx_quic_path_t *path) { + size_t mtu; ngx_int_t rc; ngx_uint_t log_error; ngx_quic_frame_t frame; @@ -895,7 +896,12 @@ ngx_quic_send_path_mtu_probe(ngx_connection_t *c, ngx_quic_path_t *path) log_error = c->log_error; c->log_error = NGX_ERROR_IGNORE_EMSGSIZE; + mtu = path->mtu; + path->mtu = path->mtud; + rc = ngx_quic_frame_sendto(c, &frame, path->mtud, path); + + path->mtu = mtu; c->log_error = log_error; if (rc == NGX_ERROR) { diff --git a/src/event/quic/ngx_event_quic_output.c b/src/event/quic/ngx_event_quic_output.c index a0acdb70023..338451e8b48 100644 --- a/src/event/quic/ngx_event_quic_output.c +++ b/src/event/quic/ngx_event_quic_output.c @@ -1181,7 +1181,7 @@ ngx_int_t ngx_quic_frame_sendto(ngx_connection_t *c, ngx_quic_frame_t *frame, size_t min, ngx_quic_path_t *path) { - size_t min_payload, pad; + size_t max, max_payload, min_payload, pad; ssize_t len, sent; ngx_str_t res; ngx_quic_header_t pkt; @@ -1194,15 +1194,25 @@ ngx_quic_frame_sendto(ngx_connection_t *c, ngx_quic_frame_t *frame, qc = ngx_quic_get_connection(c); ctx = ngx_quic_get_send_ctx(qc, frame->level); - ngx_quic_init_packet(c, ctx, &pkt, path); + max = ngx_quic_path_limit(c, path, path->mtu); - min = ngx_quic_path_limit(c, path, min); + ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic sendto %s packet max:%uz min:%uz", + ngx_quic_level_name(ctx->level), max, min); - min_payload = min ? ngx_quic_payload_size(&pkt, min) : 0; + ngx_quic_init_packet(c, ctx, &pkt, path); + + min_payload = ngx_quic_payload_size(&pkt, min); + max_payload = ngx_quic_payload_size(&pkt, max); + /* RFC 9001, 5.4.2. Header Protection Sample */ pad = 4 - pkt.num_len; min_payload = ngx_max(min_payload, pad); + if (min_payload > max_payload) { + return NGX_AGAIN; + } + #if (NGX_DEBUG) frame->pnum = pkt.number; #endif @@ -1210,8 +1220,8 @@ ngx_quic_frame_sendto(ngx_connection_t *c, ngx_quic_frame_t *frame, ngx_quic_log_frame(c->log, frame, 1); len = ngx_quic_create_frame(NULL, frame); - if (len > NGX_QUIC_MAX_UDP_PAYLOAD_SIZE) { - return NGX_ERROR; + if ((size_t) len > max_payload) { + return NGX_AGAIN; } len = ngx_quic_create_frame(src, frame); @@ -1258,8 +1268,6 @@ ngx_quic_path_limit(ngx_connection_t *c, ngx_quic_path_t *path, size_t size) max = (path->sent >= max) ? 0 : max - path->sent; if ((off_t) size > max) { - ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, - "quic path limit %uz - %O", size, max); return max; } } From 0c0f3405544404b354780ddbeacf5d54f122bcdc Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Wed, 22 Nov 2023 14:48:12 +0400 Subject: [PATCH 075/279] QUIC: ignore duplicate PATH_CHALLENGE frames. According to RFC 9000, an endpoint SHOULD NOT send multiple PATH_CHALLENGE frames in a single packet. The change adds a check to enforce this claim to optimize server behavior. Previously each PATH_CHALLENGE always resulted in a single response datagram being sent to client. The effect of this was however limited by QUIC flood protection. Also, PATH_CHALLENGE is explicitly disabled in Initial and Handshake levels, see RFC 9000, Table 3. However, technically it may be sent by client in 0-RTT over a new path without actual migration, even though the migration itself is prohibited during handshake. This allows client to coalesce multiple 0-RTT packets each carrying a PATH_CHALLENGE and end up with multiple PATH_CHALLENGEs per datagram. This again leads to suboptimal behavior, see above. Since the purpose of sending PATH_CHALLENGE frames in 0-RTT is unclear, these frames are now only allowed in 1-RTT. For 0-RTT they are silently ignored. --- src/event/quic/ngx_event_quic_migration.c | 8 ++++++++ src/event/quic/ngx_event_quic_transport.h | 1 + 2 files changed, 9 insertions(+) diff --git a/src/event/quic/ngx_event_quic_migration.c b/src/event/quic/ngx_event_quic_migration.c index efb167b0ade..a6eaf456390 100644 --- a/src/event/quic/ngx_event_quic_migration.c +++ b/src/event/quic/ngx_event_quic_migration.c @@ -40,6 +40,14 @@ ngx_quic_handle_path_challenge_frame(ngx_connection_t *c, ngx_quic_frame_t frame, *fp; ngx_quic_connection_t *qc; + if (pkt->level != ssl_encryption_application || pkt->path_challenged) { + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic ignoring PATH_CHALLENGE"); + return NGX_OK; + } + + pkt->path_challenged = 1; + qc = ngx_quic_get_connection(c); ngx_memzero(&frame, sizeof(ngx_quic_frame_t)); diff --git a/src/event/quic/ngx_event_quic_transport.h b/src/event/quic/ngx_event_quic_transport.h index 02e44650bec..232ff18d3f9 100644 --- a/src/event/quic/ngx_event_quic_transport.h +++ b/src/event/quic/ngx_event_quic_transport.h @@ -336,6 +336,7 @@ typedef struct { unsigned retried:1; unsigned first:1; unsigned rebound:1; + unsigned path_challenged:1; } ngx_quic_header_t; From ccca701dc6f3644b8d6d8d31d7094edaab703d7a Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Wed, 29 Nov 2023 21:41:29 +0400 Subject: [PATCH 076/279] QUIC: congestion control in ngx_quic_frame_sendto(). Previously ngx_quic_frame_sendto() ignored congestion control and did not contribute to in_flight counter. Now congestion control window is checked unless ignore_congestion flag is set. Also, in_flight counter is incremented and the frame is stored in ctx->sent queue if it's ack-eliciting. This behavior is now similar to ngx_quic_output_packet(). --- src/event/quic/ngx_event_quic_ack.c | 32 +++++++---- src/event/quic/ngx_event_quic_migration.c | 49 ++++++++++------- src/event/quic/ngx_event_quic_output.c | 65 +++++++++++++++++++---- src/event/quic/ngx_event_quic_transport.h | 1 + 4 files changed, 107 insertions(+), 40 deletions(-) diff --git a/src/event/quic/ngx_event_quic_ack.c b/src/event/quic/ngx_event_quic_ack.c index a7fd85d20bd..9572987e1cf 100644 --- a/src/event/quic/ngx_event_quic_ack.c +++ b/src/event/quic/ngx_event_quic_ack.c @@ -593,6 +593,7 @@ ngx_quic_resend_frames(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx) break; case NGX_QUIC_FT_PING: + case NGX_QUIC_FT_PATH_CHALLENGE: case NGX_QUIC_FT_PATH_RESPONSE: case NGX_QUIC_FT_CONNECTION_CLOSE: ngx_quic_free_frame(c, f); @@ -824,11 +825,11 @@ void ngx_quic_lost_handler(ngx_event_t *ev) void ngx_quic_pto_handler(ngx_event_t *ev) { - ngx_uint_t i; + ngx_uint_t i, n; ngx_msec_t now; ngx_queue_t *q; ngx_connection_t *c; - ngx_quic_frame_t *f, frame; + ngx_quic_frame_t *f; ngx_quic_send_ctx_t *ctx; ngx_quic_connection_t *qc; @@ -865,16 +866,20 @@ ngx_quic_pto_handler(ngx_event_t *ev) "quic pto %s pto_count:%ui", ngx_quic_level_name(ctx->level), qc->pto_count); - ngx_memzero(&frame, sizeof(ngx_quic_frame_t)); + for (n = 0; n < 2; n++) { - frame.level = ctx->level; - frame.type = NGX_QUIC_FT_PING; + f = ngx_quic_alloc_frame(c); + if (f == NULL) { + goto failed; + } - if (ngx_quic_frame_sendto(c, &frame, 0, qc->path) != NGX_OK - || ngx_quic_frame_sendto(c, &frame, 0, qc->path) != NGX_OK) - { - ngx_quic_close_connection(c, NGX_ERROR); - return; + f->level = ctx->level; + f->type = NGX_QUIC_FT_PING; + f->ignore_congestion = 1; + + if (ngx_quic_frame_sendto(c, f, 0, qc->path) == NGX_ERROR) { + goto failed; + } } } @@ -883,6 +888,13 @@ ngx_quic_pto_handler(ngx_event_t *ev) ngx_quic_set_lost_timer(c); ngx_quic_connstate_dbg(c); + + return; + +failed: + + ngx_quic_close_connection(c, NGX_ERROR); + return; } diff --git a/src/event/quic/ngx_event_quic_migration.c b/src/event/quic/ngx_event_quic_migration.c index a6eaf456390..7819ae2f9b6 100644 --- a/src/event/quic/ngx_event_quic_migration.c +++ b/src/event/quic/ngx_event_quic_migration.c @@ -37,7 +37,7 @@ ngx_quic_handle_path_challenge_frame(ngx_connection_t *c, ngx_quic_header_t *pkt, ngx_quic_path_challenge_frame_t *f) { size_t min; - ngx_quic_frame_t frame, *fp; + ngx_quic_frame_t *fp; ngx_quic_connection_t *qc; if (pkt->level != ssl_encryption_application || pkt->path_challenged) { @@ -50,11 +50,14 @@ ngx_quic_handle_path_challenge_frame(ngx_connection_t *c, qc = ngx_quic_get_connection(c); - ngx_memzero(&frame, sizeof(ngx_quic_frame_t)); + fp = ngx_quic_alloc_frame(c); + if (fp == NULL) { + return NGX_ERROR; + } - frame.level = ssl_encryption_application; - frame.type = NGX_QUIC_FT_PATH_RESPONSE; - frame.u.path_response = *f; + fp->level = ssl_encryption_application; + fp->type = NGX_QUIC_FT_PATH_RESPONSE; + fp->u.path_response = *f; /* * RFC 9000, 8.2.2. Path Validation Responses @@ -73,7 +76,7 @@ ngx_quic_handle_path_challenge_frame(ngx_connection_t *c, min = (ngx_quic_path_limit(c, pkt->path, 1200) < 1200) ? 0 : 1200; - if (ngx_quic_frame_sendto(c, &frame, min, pkt->path) == NGX_ERROR) { + if (ngx_quic_frame_sendto(c, fp, min, pkt->path) == NGX_ERROR) { return NGX_ERROR; } @@ -546,22 +549,25 @@ ngx_quic_validate_path(ngx_connection_t *c, ngx_quic_path_t *path) static ngx_int_t ngx_quic_send_path_challenge(ngx_connection_t *c, ngx_quic_path_t *path) { - size_t min; - ngx_uint_t n; - ngx_quic_frame_t frame; + size_t min; + ngx_uint_t n; + ngx_quic_frame_t *frame; ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic path seq:%uL send path_challenge tries:%ui", path->seqnum, path->tries); - ngx_memzero(&frame, sizeof(ngx_quic_frame_t)); + for (n = 0; n < 2; n++) { - frame.level = ssl_encryption_application; - frame.type = NGX_QUIC_FT_PATH_CHALLENGE; + frame = ngx_quic_alloc_frame(c); + if (frame == NULL) { + return NGX_ERROR; + } - for (n = 0; n < 2; n++) { + frame->level = ssl_encryption_application; + frame->type = NGX_QUIC_FT_PATH_CHALLENGE; - ngx_memcpy(frame.u.path_challenge.data, path->challenge[n], 8); + ngx_memcpy(frame->u.path_challenge.data, path->challenge[n], 8); /* * RFC 9000, 8.2.1. Initiating Path Validation @@ -574,7 +580,7 @@ ngx_quic_send_path_challenge(ngx_connection_t *c, ngx_quic_path_t *path) min = (ngx_quic_path_limit(c, path, 1200) < 1200) ? 0 : 1200; - if (ngx_quic_frame_sendto(c, &frame, min, path) == NGX_ERROR) { + if (ngx_quic_frame_sendto(c, frame, min, path) == NGX_ERROR) { return NGX_ERROR; } } @@ -883,14 +889,17 @@ ngx_quic_send_path_mtu_probe(ngx_connection_t *c, ngx_quic_path_t *path) size_t mtu; ngx_int_t rc; ngx_uint_t log_error; - ngx_quic_frame_t frame; + ngx_quic_frame_t *frame; ngx_quic_send_ctx_t *ctx; ngx_quic_connection_t *qc; - ngx_memzero(&frame, sizeof(ngx_quic_frame_t)); + frame = ngx_quic_alloc_frame(c); + if (frame == NULL) { + return NGX_ERROR; + } - frame.level = ssl_encryption_application; - frame.type = NGX_QUIC_FT_PING; + frame->level = ssl_encryption_application; + frame->type = NGX_QUIC_FT_PING; qc = ngx_quic_get_connection(c); ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_application); @@ -907,7 +916,7 @@ ngx_quic_send_path_mtu_probe(ngx_connection_t *c, ngx_quic_path_t *path) mtu = path->mtu; path->mtu = path->mtud; - rc = ngx_quic_frame_sendto(c, &frame, path->mtud, path); + rc = ngx_quic_frame_sendto(c, frame, path->mtud, path); path->mtu = mtu; c->log_error = log_error; diff --git a/src/event/quic/ngx_event_quic_output.c b/src/event/quic/ngx_event_quic_output.c index 338451e8b48..1497b783176 100644 --- a/src/event/quic/ngx_event_quic_output.c +++ b/src/event/quic/ngx_event_quic_output.c @@ -844,7 +844,7 @@ ngx_quic_send_stateless_reset(ngx_connection_t *c, ngx_quic_conf_t *conf, ngx_int_t ngx_quic_send_cc(ngx_connection_t *c) { - ngx_quic_frame_t frame; + ngx_quic_frame_t *frame; ngx_quic_connection_t *qc; qc = ngx_quic_get_connection(c); @@ -860,22 +860,27 @@ ngx_quic_send_cc(ngx_connection_t *c) return NGX_OK; } - ngx_memzero(&frame, sizeof(ngx_quic_frame_t)); + frame = ngx_quic_alloc_frame(c); + if (frame == NULL) { + return NGX_ERROR; + } - frame.level = qc->error_level; - frame.type = qc->error_app ? NGX_QUIC_FT_CONNECTION_CLOSE_APP - : NGX_QUIC_FT_CONNECTION_CLOSE; - frame.u.close.error_code = qc->error; - frame.u.close.frame_type = qc->error_ftype; + frame->level = qc->error_level; + frame->type = qc->error_app ? NGX_QUIC_FT_CONNECTION_CLOSE_APP + : NGX_QUIC_FT_CONNECTION_CLOSE; + frame->u.close.error_code = qc->error; + frame->u.close.frame_type = qc->error_ftype; if (qc->error_reason) { - frame.u.close.reason.len = ngx_strlen(qc->error_reason); - frame.u.close.reason.data = (u_char *) qc->error_reason; + frame->u.close.reason.len = ngx_strlen(qc->error_reason); + frame->u.close.reason.data = (u_char *) qc->error_reason; } + frame->ignore_congestion = 1; + qc->last_cc = ngx_current_msec; - return ngx_quic_frame_sendto(c, &frame, 0, qc->path); + return ngx_quic_frame_sendto(c, frame, 0, qc->path); } @@ -1184,22 +1189,32 @@ ngx_quic_frame_sendto(ngx_connection_t *c, ngx_quic_frame_t *frame, size_t max, max_payload, min_payload, pad; ssize_t len, sent; ngx_str_t res; + ngx_msec_t now; ngx_quic_header_t pkt; ngx_quic_send_ctx_t *ctx; + ngx_quic_congestion_t *cg; ngx_quic_connection_t *qc; static u_char src[NGX_QUIC_MAX_UDP_PAYLOAD_SIZE]; static u_char dst[NGX_QUIC_MAX_UDP_PAYLOAD_SIZE]; qc = ngx_quic_get_connection(c); + cg = &qc->congestion; ctx = ngx_quic_get_send_ctx(qc, frame->level); + now = ngx_current_msec; + max = ngx_quic_path_limit(c, path, path->mtu); ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic sendto %s packet max:%uz min:%uz", ngx_quic_level_name(ctx->level), max, min); + if (cg->in_flight >= cg->window && !frame->ignore_congestion) { + ngx_quic_free_frame(c, frame); + return NGX_AGAIN; + } + ngx_quic_init_packet(c, ctx, &pkt, path); min_payload = ngx_quic_payload_size(&pkt, min); @@ -1210,6 +1225,7 @@ ngx_quic_frame_sendto(ngx_connection_t *c, ngx_quic_frame_t *frame, min_payload = ngx_max(min_payload, pad); if (min_payload > max_payload) { + ngx_quic_free_frame(c, frame); return NGX_AGAIN; } @@ -1221,11 +1237,13 @@ ngx_quic_frame_sendto(ngx_connection_t *c, ngx_quic_frame_t *frame, len = ngx_quic_create_frame(NULL, frame); if ((size_t) len > max_payload) { + ngx_quic_free_frame(c, frame); return NGX_AGAIN; } len = ngx_quic_create_frame(src, frame); if (len == -1) { + ngx_quic_free_frame(c, frame); return NGX_ERROR; } @@ -1242,18 +1260,45 @@ ngx_quic_frame_sendto(ngx_connection_t *c, ngx_quic_frame_t *frame, ngx_quic_log_packet(c->log, &pkt); if (ngx_quic_encrypt(&pkt, &res) != NGX_OK) { + ngx_quic_free_frame(c, frame); return NGX_ERROR; } + frame->pnum = ctx->pnum; + frame->first = now; + frame->last = now; + frame->plen = res.len; + ctx->pnum++; sent = ngx_quic_send(c, res.data, res.len, path->sockaddr, path->socklen); if (sent < 0) { + ngx_quic_free_frame(c, frame); return sent; } path->sent += sent; + if (frame->need_ack && !qc->closing) { + ngx_queue_insert_tail(&ctx->sent, &frame->queue); + + cg->in_flight += frame->plen; + + } else { + ngx_quic_free_frame(c, frame); + return NGX_OK; + } + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic congestion send if:%uz", cg->in_flight); + + if (!qc->send_timer_set) { + qc->send_timer_set = 1; + ngx_add_timer(c->read, qc->tp.max_idle_timeout); + } + + ngx_quic_set_lost_timer(c); + return NGX_OK; } diff --git a/src/event/quic/ngx_event_quic_transport.h b/src/event/quic/ngx_event_quic_transport.h index 232ff18d3f9..9891ede44b5 100644 --- a/src/event/quic/ngx_event_quic_transport.h +++ b/src/event/quic/ngx_event_quic_transport.h @@ -271,6 +271,7 @@ struct ngx_quic_frame_s { ssize_t len; unsigned need_ack:1; unsigned pkt_need_ack:1; + unsigned ignore_congestion:1; ngx_chain_t *data; union { From 209e8bc0c04f852605919ed766d25ee777f4b52b Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Thu, 30 Nov 2023 15:03:06 +0400 Subject: [PATCH 077/279] QUIC: ngx_quic_frame_t time fields cleanup. The field "first" is removed. It's unused since 909b989ec088. The field "last" is renamed to "send_time". It holds frame send time. --- src/event/quic/ngx_event_quic_ack.c | 40 ++++++++++++----------- src/event/quic/ngx_event_quic_output.c | 6 ++-- src/event/quic/ngx_event_quic_transport.h | 3 +- 3 files changed, 24 insertions(+), 25 deletions(-) diff --git a/src/event/quic/ngx_event_quic_ack.c b/src/event/quic/ngx_event_quic_ack.c index 9572987e1cf..deeaae1e343 100644 --- a/src/event/quic/ngx_event_quic_ack.c +++ b/src/event/quic/ngx_event_quic_ack.c @@ -265,16 +265,16 @@ ngx_quic_handle_ack_frame_range(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx, } if (f->pnum == max) { - st->max_pn = f->last; + st->max_pn = f->send_time; } /* save earliest and latest send times of frames ack'ed */ - if (st->oldest == NGX_TIMER_INFINITE || f->last < st->oldest) { - st->oldest = f->last; + if (st->oldest == NGX_TIMER_INFINITE || f->send_time < st->oldest) { + st->oldest = f->send_time; } - if (st->newest == NGX_TIMER_INFINITE || f->last > st->newest) { - st->newest = f->last; + if (st->newest == NGX_TIMER_INFINITE || f->send_time > st->newest) { + st->newest = f->send_time; } ngx_queue_remove(&f->queue); @@ -329,7 +329,7 @@ ngx_quic_congestion_ack(ngx_connection_t *c, ngx_quic_frame_t *f) cg->in_flight -= f->plen; - timer = f->last - cg->recovery_start; + timer = f->send_time - cg->recovery_start; if ((ngx_msec_int_t) timer <= 0) { ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0, @@ -465,7 +465,7 @@ ngx_quic_detect_lost(ngx_connection_t *c, ngx_quic_ack_stat_t *st) break; } - wait = start->last + thr - now; + wait = start->send_time + thr - now; ngx_log_debug4(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic detect_lost pnum:%uL thr:%M wait:%i level:%d", @@ -477,14 +477,14 @@ ngx_quic_detect_lost(ngx_connection_t *c, ngx_quic_ack_stat_t *st) break; } - if (start->last > qc->first_rtt) { + if (start->send_time > qc->first_rtt) { - if (oldest == NGX_TIMER_INFINITE || start->last < oldest) { - oldest = start->last; + if (oldest == NGX_TIMER_INFINITE || start->send_time < oldest) { + oldest = start->send_time; } - if (newest == NGX_TIMER_INFINITE || start->last > newest) { - newest = start->last; + if (newest == NGX_TIMER_INFINITE || start->send_time > newest) { + newest = start->send_time; } nlost++; @@ -672,7 +672,7 @@ ngx_quic_congestion_lost(ngx_connection_t *c, ngx_quic_frame_t *f) cg->in_flight -= f->plen; f->plen = 0; - timer = f->last - cg->recovery_start; + timer = f->send_time - cg->recovery_start; if ((ngx_msec_int_t) timer <= 0) { ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0, @@ -730,7 +730,8 @@ ngx_quic_set_lost_timer(ngx_connection_t *c) if (ctx->largest_ack != NGX_QUIC_UNSET_PN) { q = ngx_queue_head(&ctx->sent); f = ngx_queue_data(q, ngx_quic_frame_t, queue); - w = (ngx_msec_int_t) (f->last + ngx_quic_lost_threshold(qc) - now); + w = (ngx_msec_int_t) + (f->send_time + ngx_quic_lost_threshold(qc) - now); if (f->pnum <= ctx->largest_ack) { if (w < 0 || ctx->largest_ack - f->pnum >= NGX_QUIC_PKT_THR) { @@ -745,8 +746,8 @@ ngx_quic_set_lost_timer(ngx_connection_t *c) q = ngx_queue_last(&ctx->sent); f = ngx_queue_data(q, ngx_quic_frame_t, queue); - w = (ngx_msec_int_t) (f->last + (ngx_quic_pto(c, ctx) << qc->pto_count) - - now); + w = (ngx_msec_int_t) + (f->send_time + (ngx_quic_pto(c, ctx) << qc->pto_count) - now); if (w < 0) { w = 0; @@ -828,6 +829,7 @@ ngx_quic_pto_handler(ngx_event_t *ev) ngx_uint_t i, n; ngx_msec_t now; ngx_queue_t *q; + ngx_msec_int_t w; ngx_connection_t *c; ngx_quic_frame_t *f; ngx_quic_send_ctx_t *ctx; @@ -849,6 +851,8 @@ ngx_quic_pto_handler(ngx_event_t *ev) q = ngx_queue_last(&ctx->sent); f = ngx_queue_data(q, ngx_quic_frame_t, queue); + w = (ngx_msec_int_t) + (f->send_time + (ngx_quic_pto(c, ctx) << qc->pto_count) - now); if (f->pnum <= ctx->largest_ack && ctx->largest_ack != NGX_QUIC_UNSET_PN) @@ -856,9 +860,7 @@ ngx_quic_pto_handler(ngx_event_t *ev) continue; } - if ((ngx_msec_int_t) (f->last + (ngx_quic_pto(c, ctx) << qc->pto_count) - - now) > 0) - { + if (w > 0) { continue; } diff --git a/src/event/quic/ngx_event_quic_output.c b/src/event/quic/ngx_event_quic_output.c index 1497b783176..ce6aaab227d 100644 --- a/src/event/quic/ngx_event_quic_output.c +++ b/src/event/quic/ngx_event_quic_output.c @@ -586,8 +586,7 @@ ngx_quic_output_packet(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx, } f->pnum = ctx->pnum; - f->first = now; - f->last = now; + f->send_time = now; f->plen = 0; ngx_quic_log_frame(c->log, f, 1); @@ -1265,8 +1264,7 @@ ngx_quic_frame_sendto(ngx_connection_t *c, ngx_quic_frame_t *frame, } frame->pnum = ctx->pnum; - frame->first = now; - frame->last = now; + frame->send_time = now; frame->plen = res.len; ctx->pnum++; diff --git a/src/event/quic/ngx_event_quic_transport.h b/src/event/quic/ngx_event_quic_transport.h index 9891ede44b5..3e320391afc 100644 --- a/src/event/quic/ngx_event_quic_transport.h +++ b/src/event/quic/ngx_event_quic_transport.h @@ -266,8 +266,7 @@ struct ngx_quic_frame_s { ngx_queue_t queue; uint64_t pnum; size_t plen; - ngx_msec_t first; - ngx_msec_t last; + ngx_msec_t send_time; ssize_t len; unsigned need_ack:1; unsigned pkt_need_ack:1; From c1efb3a725c189dc0f011753561bf34ff650d773 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Wed, 29 Nov 2023 10:58:21 +0400 Subject: [PATCH 078/279] QUIC: path revalidation after expansion failure. As per RFC 9000, Section 8.2.1: When an endpoint is unable to expand the datagram size to 1200 bytes due to the anti-amplification limit, the path MTU will not be validated. To ensure that the path MTU is large enough, the endpoint MUST perform a second path validation by sending a PATH_CHALLENGE frame in a datagram of at least 1200 bytes. --- src/event/quic/ngx_event_quic_connection.h | 3 ++- src/event/quic/ngx_event_quic_migration.c | 20 +++++++++++++++++--- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/event/quic/ngx_event_quic_connection.h b/src/event/quic/ngx_event_quic_connection.h index abd0ebe6cc9..4ff0eae4a37 100644 --- a/src/event/quic/ngx_event_quic_connection.h +++ b/src/event/quic/ngx_event_quic_connection.h @@ -111,7 +111,8 @@ struct ngx_quic_path_s { uint64_t mtu_pnum[NGX_QUIC_PATH_RETRIES]; ngx_str_t addr_text; u_char text[NGX_SOCKADDR_STRLEN]; - ngx_uint_t validated; /* unsigned validated:1; */ + unsigned validated:1; + unsigned mtu_unvalidated:1; }; diff --git a/src/event/quic/ngx_event_quic_migration.c b/src/event/quic/ngx_event_quic_migration.c index 7819ae2f9b6..be8e0e30484 100644 --- a/src/event/quic/ngx_event_quic_migration.c +++ b/src/event/quic/ngx_event_quic_migration.c @@ -169,6 +169,7 @@ ngx_quic_handle_path_response_frame(ngx_connection_t *c, path->mtu = prev->mtu; path->max_mtu = prev->max_mtu; + path->mtu_unvalidated = 0; } } @@ -182,6 +183,13 @@ ngx_quic_handle_path_response_frame(ngx_connection_t *c, qc->congestion.recovery_start = ngx_current_msec; } + path->validated = 1; + + if (path->mtu_unvalidated) { + path->mtu_unvalidated = 0; + return ngx_quic_validate_path(c, path); + } + /* * RFC 9000, 9.3. Responding to Connection Migration * @@ -199,8 +207,6 @@ ngx_quic_handle_path_response_frame(ngx_connection_t *c, ngx_quic_path_dbg(c, "is validated", path); - path->validated = 1; - ngx_quic_discover_path_mtu(c, path); return NGX_OK; @@ -578,7 +584,15 @@ ngx_quic_send_path_challenge(ngx_connection_t *c, ngx_quic_path_t *path) * sending a datagram of this size. */ - min = (ngx_quic_path_limit(c, path, 1200) < 1200) ? 0 : 1200; + if (path->mtu_unvalidated + || ngx_quic_path_limit(c, path, 1200) < 1200) + { + min = 0; + path->mtu_unvalidated = 1; + + } else { + min = 1200; + } if (ngx_quic_frame_sendto(c, frame, min, path) == NGX_ERROR) { return NGX_ERROR; From 4ee2a48f3f230b0efe80ff84bd58d7ab209a46a9 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Tue, 12 Dec 2023 20:20:51 +0400 Subject: [PATCH 079/279] QUIC: reset RTT estimator for the new path. RTT is a property of the path, it must be reset on confirming a peer's ownership of its new address. --- src/event/quic/ngx_event_quic.c | 9 +-------- src/event/quic/ngx_event_quic_connection.h | 7 +++++++ src/event/quic/ngx_event_quic_migration.c | 2 ++ 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/event/quic/ngx_event_quic.c b/src/event/quic/ngx_event_quic.c index b0cf056c1ee..4687c849e42 100644 --- a/src/event/quic/ngx_event_quic.c +++ b/src/event/quic/ngx_event_quic.c @@ -260,14 +260,7 @@ ngx_quic_new_connection(ngx_connection_t *c, ngx_quic_conf_t *conf, ngx_queue_init(&qc->free_frames); - qc->avg_rtt = NGX_QUIC_INITIAL_RTT; - qc->rttvar = NGX_QUIC_INITIAL_RTT / 2; - qc->min_rtt = NGX_TIMER_INFINITE; - qc->first_rtt = NGX_TIMER_INFINITE; - - /* - * qc->latest_rtt = 0 - */ + ngx_quic_init_rtt(qc); qc->pto.log = c->log; qc->pto.data = c; diff --git a/src/event/quic/ngx_event_quic_connection.h b/src/event/quic/ngx_event_quic_connection.h index 4ff0eae4a37..ae771bcc59e 100644 --- a/src/event/quic/ngx_event_quic_connection.h +++ b/src/event/quic/ngx_event_quic_connection.h @@ -65,6 +65,13 @@ typedef struct ngx_quic_keys_s ngx_quic_keys_t; #define ngx_quic_get_socket(c) ((ngx_quic_socket_t *)((c)->udp)) +#define ngx_quic_init_rtt(qc) \ + (qc)->avg_rtt = NGX_QUIC_INITIAL_RTT; \ + (qc)->rttvar = NGX_QUIC_INITIAL_RTT / 2; \ + (qc)->min_rtt = NGX_TIMER_INFINITE; \ + (qc)->first_rtt = NGX_TIMER_INFINITE; \ + (qc)->latest_rtt = 0; + typedef enum { NGX_QUIC_PATH_IDLE = 0, diff --git a/src/event/quic/ngx_event_quic_migration.c b/src/event/quic/ngx_event_quic_migration.c index be8e0e30484..58f3fc545a8 100644 --- a/src/event/quic/ngx_event_quic_migration.c +++ b/src/event/quic/ngx_event_quic_migration.c @@ -181,6 +181,8 @@ ngx_quic_handle_path_response_frame(ngx_connection_t *c, 14720)); qc->congestion.ssthresh = (size_t) -1; qc->congestion.recovery_start = ngx_current_msec; + + ngx_quic_init_rtt(qc); } path->validated = 1; From 386329d3cf207e0a52056d195fefb4e85dfc8c26 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Tue, 12 Dec 2023 20:21:12 +0400 Subject: [PATCH 080/279] QUIC: path aware in-flight bytes accounting. On-packet acknowledgement is made path aware, as per RFC 9000, Section 9.4: Packets sent on the old path MUST NOT contribute to congestion control or RTT estimation for the new path. To make this possible in a single congestion control context, the first packet to be sent after the new path has been validated, which includes resetting the congestion controller and RTT estimator, is now remembered in the connection. Packets sent previously, such as on the old path, are not taken into account. Note that although the packet number is saved per-connection, the added checks affect application level packets only. For non-application level packets, which are only processed prior to the handshake is complete, the remembered packet number remains set to zero. --- src/event/quic/ngx_event_quic_ack.c | 8 ++++++++ src/event/quic/ngx_event_quic_connection.h | 2 ++ src/event/quic/ngx_event_quic_migration.c | 6 ++++++ 3 files changed, 16 insertions(+) diff --git a/src/event/quic/ngx_event_quic_ack.c b/src/event/quic/ngx_event_quic_ack.c index deeaae1e343..c7ffd44dd77 100644 --- a/src/event/quic/ngx_event_quic_ack.c +++ b/src/event/quic/ngx_event_quic_ack.c @@ -325,6 +325,10 @@ ngx_quic_congestion_ack(ngx_connection_t *c, ngx_quic_frame_t *f) qc = ngx_quic_get_connection(c); cg = &qc->congestion; + if (f->pnum < qc->rst_pnum) { + return; + } + blocked = (cg->in_flight >= cg->window) ? 1 : 0; cg->in_flight -= f->plen; @@ -667,6 +671,10 @@ ngx_quic_congestion_lost(ngx_connection_t *c, ngx_quic_frame_t *f) qc = ngx_quic_get_connection(c); cg = &qc->congestion; + if (f->pnum < qc->rst_pnum) { + return; + } + blocked = (cg->in_flight >= cg->window) ? 1 : 0; cg->in_flight -= f->plen; diff --git a/src/event/quic/ngx_event_quic_connection.h b/src/event/quic/ngx_event_quic_connection.h index ae771bcc59e..824c92b5791 100644 --- a/src/event/quic/ngx_event_quic_connection.h +++ b/src/event/quic/ngx_event_quic_connection.h @@ -266,6 +266,8 @@ struct ngx_quic_connection_s { ngx_quic_streams_t streams; ngx_quic_congestion_t congestion; + uint64_t rst_pnum; /* first on validated path */ + off_t received; ngx_uint_t error; diff --git a/src/event/quic/ngx_event_quic_migration.c b/src/event/quic/ngx_event_quic_migration.c index 58f3fc545a8..3c1bbaf43f0 100644 --- a/src/event/quic/ngx_event_quic_migration.c +++ b/src/event/quic/ngx_event_quic_migration.c @@ -110,6 +110,7 @@ ngx_quic_handle_path_response_frame(ngx_connection_t *c, ngx_uint_t rst; ngx_queue_t *q; ngx_quic_path_t *path, *prev; + ngx_quic_send_ctx_t *ctx; ngx_quic_connection_t *qc; qc = ngx_quic_get_connection(c); @@ -174,6 +175,11 @@ ngx_quic_handle_path_response_frame(ngx_connection_t *c, } if (rst) { + /* prevent old path packets contribution to congestion control */ + + ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_application); + qc->rst_pnum = ctx->pnum; + ngx_memzero(&qc->congestion, sizeof(ngx_quic_congestion_t)); qc->congestion.window = ngx_min(10 * qc->tp.max_udp_payload_size, From 5e743242849f4724392c742b59e3757e5c7ea499 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Sat, 16 Dec 2023 03:40:01 +0400 Subject: [PATCH 081/279] QUIC: fixed format specifier after a6f79f044de5. --- src/event/quic/ngx_event_quic_migration.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/event/quic/ngx_event_quic_migration.h b/src/event/quic/ngx_event_quic_migration.h index 9587882f280..270c572ed86 100644 --- a/src/event/quic/ngx_event_quic_migration.h +++ b/src/event/quic/ngx_event_quic_migration.h @@ -19,7 +19,7 @@ #define ngx_quic_path_dbg(c, msg, path) \ ngx_log_debug7(NGX_LOG_DEBUG_EVENT, c->log, 0, \ - "quic path seq:%uL %s tx:%O rx:%O valid:%ui st:%d mtu:%uz",\ + "quic path seq:%uL %s tx:%O rx:%O valid:%d st:%d mtu:%uz", \ path->seqnum, msg, path->sent, path->received, \ path->validated, path->state, path->mtu); From c0134ded9fce48f36465cb1a1adbcc52e32f8c65 Mon Sep 17 00:00:00 2001 From: J Carter Date: Sat, 9 Dec 2023 08:38:14 +0000 Subject: [PATCH 082/279] Win32: extended ngx_random() range to 0x7fffffff. rand() is used on win32. RAND_MAX is implementation defined. win32's is 0x7fff. Existing uses of ngx_random() rely upon 0x7fffffff range provided by POSIX implementations of random(). --- src/os/win32/ngx_win32_config.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/os/win32/ngx_win32_config.h b/src/os/win32/ngx_win32_config.h index 406003a7804..704561355c2 100644 --- a/src/os/win32/ngx_win32_config.h +++ b/src/os/win32/ngx_win32_config.h @@ -280,7 +280,11 @@ typedef int sig_atomic_t; #define NGX_HAVE_GETADDRINFO 1 -#define ngx_random rand +#define ngx_random() \ + ((long) (0x7fffffff & ( ((uint32_t) rand() << 16) \ + ^ ((uint32_t) rand() << 8) \ + ^ ((uint32_t) rand()) ))) + #define ngx_debug_init() From d7923960a824d0dd9f4b0c35101e68d71692d1e9 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Mon, 25 Dec 2023 21:15:47 +0400 Subject: [PATCH 083/279] SSL: disabled renegotiation checks with LibreSSL. Similar to 7356:e3ba4026c02d, as long as SSL_OP_NO_CLIENT_RENEGOTIATION is defined, it is the library responsibility to prevent renegotiation. Additionally, this allows to raise LibreSSL version used to redefine OPENSSL_VERSION_NUMBER to 0x1010000fL, such that this won't result in attempts to dereference SSL objects made opaque in LibreSSL 3.4.0. Patch by Maxim Dounin. --- src/event/ngx_event_openssl.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c index 8468101d1f2..104dd03f274 100644 --- a/src/event/ngx_event_openssl.c +++ b/src/event/ngx_event_openssl.c @@ -1105,7 +1105,8 @@ ngx_ssl_info_callback(const ngx_ssl_conn_t *ssl_conn, int where, int ret) BIO *rbio, *wbio; ngx_connection_t *c; -#ifndef SSL_OP_NO_RENEGOTIATION +#if (!defined SSL_OP_NO_RENEGOTIATION \ + && !defined SSL_OP_NO_CLIENT_RENEGOTIATION) if ((where & SSL_CB_HANDSHAKE_START) && SSL_is_server((ngx_ssl_conn_t *) ssl_conn)) @@ -1838,9 +1839,10 @@ ngx_ssl_handshake(ngx_connection_t *c) c->read->ready = 1; c->write->ready = 1; -#ifndef SSL_OP_NO_RENEGOTIATION -#if OPENSSL_VERSION_NUMBER < 0x10100000L -#ifdef SSL3_FLAGS_NO_RENEGOTIATE_CIPHERS +#if (!defined SSL_OP_NO_RENEGOTIATION \ + && !defined SSL_OP_NO_CLIENT_RENEGOTIATION \ + && defined SSL3_FLAGS_NO_RENEGOTIATE_CIPHERS \ + && OPENSSL_VERSION_NUMBER < 0x10100000L) /* initial handshake done, disable renegotiation (CVE-2009-3555) */ if (c->ssl->connection->s3 && SSL_is_server(c->ssl->connection)) { @@ -1848,8 +1850,6 @@ ngx_ssl_handshake(ngx_connection_t *c) } #endif -#endif -#endif #if (defined BIO_get_ktls_send && !NGX_WIN32) @@ -2483,7 +2483,8 @@ ngx_ssl_handle_recv(ngx_connection_t *c, int n) int sslerr; ngx_err_t err; -#ifndef SSL_OP_NO_RENEGOTIATION +#if (!defined SSL_OP_NO_RENEGOTIATION \ + && !defined SSL_OP_NO_CLIENT_RENEGOTIATION) if (c->ssl->renegotiation) { /* From f255815f5d161fab0dd310fe826d4f7572e141f2 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Mon, 25 Dec 2023 21:15:48 +0400 Subject: [PATCH 084/279] SSL: reasonable version for LibreSSL adjusted. OPENSSL_VERSION_NUMBER is now redefined to 0x1010000fL for LibreSSL 3.5.0 and above. Building with older LibreSSL versions, such as 2.8.0, may now produce warnings (see cab37803ebb3) and may require appropriate compiler options to suppress them. Notably, this allows to start using SSL_get0_verified_chain() appeared in OpenSSL 1.1.0 and LibreSSL 3.5.0, without additional macro tests. Prodded by Ilya Shipitsin. --- src/event/ngx_event_openssl.h | 2 +- src/event/ngx_event_openssl_stapling.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h index c062f912c24..ebb2c35bf28 100644 --- a/src/event/ngx_event_openssl.h +++ b/src/event/ngx_event_openssl.h @@ -45,7 +45,7 @@ #if (defined LIBRESSL_VERSION_NUMBER && OPENSSL_VERSION_NUMBER == 0x20000000L) #undef OPENSSL_VERSION_NUMBER -#if (LIBRESSL_VERSION_NUMBER >= 0x2080000fL) +#if (LIBRESSL_VERSION_NUMBER >= 0x3050000fL) #define OPENSSL_VERSION_NUMBER 0x1010000fL #else #define OPENSSL_VERSION_NUMBER 0x1000107fL diff --git a/src/event/ngx_event_openssl_stapling.c b/src/event/ngx_event_openssl_stapling.c index e3fa8c4e2be..e9bb8354eee 100644 --- a/src/event/ngx_event_openssl_stapling.c +++ b/src/event/ngx_event_openssl_stapling.c @@ -893,7 +893,7 @@ ngx_ssl_ocsp_validate(ngx_connection_t *c) ocsp->cert_status = V_OCSP_CERTSTATUS_GOOD; ocsp->conf = ocf; -#if (OPENSSL_VERSION_NUMBER >= 0x10100000L && !defined LIBRESSL_VERSION_NUMBER) +#if OPENSSL_VERSION_NUMBER >= 0x10100000L ocsp->certs = SSL_get0_verified_chain(c->ssl->connection); From cc4c3ee0a45ca3a2eeac2874e9199bf42560015c Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Mon, 29 Jan 2024 10:29:39 +0300 Subject: [PATCH 085/279] Silenced complaints about socket leaks on forced termination. When graceful shutdown was requested, and then nginx was forced to do fast shutdown, it used to (incorrectly) complain about open sockets left in connections which weren't yet closed when fast shutdown was requested. Fix is to avoid complaining about open sockets when fast shutdown was requested after graceful one. Abnormal termination, if requested with the WINCH signal, can still happen though. --- src/os/unix/ngx_process_cycle.c | 10 +++++----- src/os/win32/ngx_process_cycle.c | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/os/unix/ngx_process_cycle.c b/src/os/unix/ngx_process_cycle.c index 98d2dd29b14..5bc5ce9792d 100644 --- a/src/os/unix/ngx_process_cycle.c +++ b/src/os/unix/ngx_process_cycle.c @@ -948,7 +948,7 @@ ngx_worker_process_exit(ngx_cycle_t *cycle) } } - if (ngx_exiting) { + if (ngx_exiting && !ngx_terminate) { c = cycle->connections; for (i = 0; i < cycle->connection_n; i++) { if (c[i].fd != -1 @@ -963,11 +963,11 @@ ngx_worker_process_exit(ngx_cycle_t *cycle) ngx_debug_quit = 1; } } + } - if (ngx_debug_quit) { - ngx_log_error(NGX_LOG_ALERT, cycle->log, 0, "aborting"); - ngx_debug_point(); - } + if (ngx_debug_quit) { + ngx_log_error(NGX_LOG_ALERT, cycle->log, 0, "aborting"); + ngx_debug_point(); } /* diff --git a/src/os/win32/ngx_process_cycle.c b/src/os/win32/ngx_process_cycle.c index 0c848eff4f8..a39335fd16f 100644 --- a/src/os/win32/ngx_process_cycle.c +++ b/src/os/win32/ngx_process_cycle.c @@ -834,7 +834,7 @@ ngx_worker_process_exit(ngx_cycle_t *cycle) } } - if (ngx_exiting) { + if (ngx_exiting && !ngx_terminate) { c = cycle->connections; for (i = 0; i < cycle->connection_n; i++) { if (c[i].fd != (ngx_socket_t) -1 From b794465178ecf48f13b37e4397145b05fd3fe6df Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Mon, 29 Jan 2024 10:31:37 +0300 Subject: [PATCH 086/279] AIO operations now add timers (ticket #2162). Each AIO (thread IO) operation being run is now accompanied with 1-minute timer. This timer prevents unexpected shutdown of the worker process while an AIO operation is running, and logs an alert if the operation is running for too long. This fixes "open socket left" alerts during worker processes shutdown due to pending AIO (or thread IO) operations while corresponding requests have no timers. In particular, such errors were observed while reading cache headers (ticket #2162), and with worker_shutdown_timeout. --- src/http/ngx_http_copy_filter_module.c | 26 ++++++++++++++++++++++++++ src/http/ngx_http_file_cache.c | 26 ++++++++++++++++++++++++++ src/http/ngx_http_upstream.c | 13 +++++++++++++ src/os/unix/ngx_files.c | 4 ++++ src/os/unix/ngx_linux_sendfile_chain.c | 1 + 5 files changed, 70 insertions(+) diff --git a/src/http/ngx_http_copy_filter_module.c b/src/http/ngx_http_copy_filter_module.c index bd3028bc768..6a96e003f2c 100644 --- a/src/http/ngx_http_copy_filter_module.c +++ b/src/http/ngx_http_copy_filter_module.c @@ -170,6 +170,8 @@ ngx_http_copy_aio_handler(ngx_output_chain_ctx_t *ctx, ngx_file_t *file) file->aio->data = r; file->aio->handler = ngx_http_copy_aio_event_handler; + ngx_add_timer(&file->aio->event, 60000); + r->main->blocked++; r->aio = 1; ctx->aio = 1; @@ -192,6 +194,17 @@ ngx_http_copy_aio_event_handler(ngx_event_t *ev) ngx_log_debug2(NGX_LOG_DEBUG_HTTP, c->log, 0, "http aio: \"%V?%V\"", &r->uri, &r->args); + if (ev->timedout) { + ngx_log_error(NGX_LOG_ALERT, c->log, 0, + "aio operation took too long"); + ev->timedout = 0; + return; + } + + if (ev->timer_set) { + ngx_del_timer(ev); + } + r->main->blocked--; r->aio = 0; @@ -264,6 +277,8 @@ ngx_http_copy_thread_handler(ngx_thread_task_t *task, ngx_file_t *file) return NGX_ERROR; } + ngx_add_timer(&task->event, 60000); + r->main->blocked++; r->aio = 1; @@ -288,6 +303,17 @@ ngx_http_copy_thread_event_handler(ngx_event_t *ev) ngx_log_debug2(NGX_LOG_DEBUG_HTTP, c->log, 0, "http thread: \"%V?%V\"", &r->uri, &r->args); + if (ev->timedout) { + ngx_log_error(NGX_LOG_ALERT, c->log, 0, + "thread operation took too long"); + ev->timedout = 0; + return; + } + + if (ev->timer_set) { + ngx_del_timer(ev); + } + r->main->blocked--; r->aio = 0; diff --git a/src/http/ngx_http_file_cache.c b/src/http/ngx_http_file_cache.c index aa5fd191725..6c49dd42df9 100644 --- a/src/http/ngx_http_file_cache.c +++ b/src/http/ngx_http_file_cache.c @@ -690,6 +690,8 @@ ngx_http_file_cache_aio_read(ngx_http_request_t *r, ngx_http_cache_t *c) c->file.aio->data = r; c->file.aio->handler = ngx_http_cache_aio_event_handler; + ngx_add_timer(&c->file.aio->event, 60000); + r->main->blocked++; r->aio = 1; @@ -737,6 +739,17 @@ ngx_http_cache_aio_event_handler(ngx_event_t *ev) ngx_log_debug2(NGX_LOG_DEBUG_HTTP, c->log, 0, "http file cache aio: \"%V?%V\"", &r->uri, &r->args); + if (ev->timedout) { + ngx_log_error(NGX_LOG_ALERT, c->log, 0, + "aio operation took too long"); + ev->timedout = 0; + return; + } + + if (ev->timer_set) { + ngx_del_timer(ev); + } + r->main->blocked--; r->aio = 0; @@ -786,6 +799,8 @@ ngx_http_cache_thread_handler(ngx_thread_task_t *task, ngx_file_t *file) return NGX_ERROR; } + ngx_add_timer(&task->event, 60000); + r->main->blocked++; r->aio = 1; @@ -807,6 +822,17 @@ ngx_http_cache_thread_event_handler(ngx_event_t *ev) ngx_log_debug2(NGX_LOG_DEBUG_HTTP, c->log, 0, "http file cache thread: \"%V?%V\"", &r->uri, &r->args); + if (ev->timedout) { + ngx_log_error(NGX_LOG_ALERT, c->log, 0, + "thread operation took too long"); + ev->timedout = 0; + return; + } + + if (ev->timer_set) { + ngx_del_timer(ev); + } + r->main->blocked--; r->aio = 0; diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c index f5db65338da..99215ad8270 100644 --- a/src/http/ngx_http_upstream.c +++ b/src/http/ngx_http_upstream.c @@ -3949,6 +3949,8 @@ ngx_http_upstream_thread_handler(ngx_thread_task_t *task, ngx_file_t *file) r->aio = 1; p->aio = 1; + ngx_add_timer(&task->event, 60000); + return NGX_OK; } @@ -3967,6 +3969,17 @@ ngx_http_upstream_thread_event_handler(ngx_event_t *ev) ngx_log_debug2(NGX_LOG_DEBUG_HTTP, c->log, 0, "http upstream thread: \"%V?%V\"", &r->uri, &r->args); + if (ev->timedout) { + ngx_log_error(NGX_LOG_ALERT, c->log, 0, + "thread operation took too long"); + ev->timedout = 0; + return; + } + + if (ev->timer_set) { + ngx_del_timer(ev); + } + r->main->blocked--; r->aio = 0; diff --git a/src/os/unix/ngx_files.c b/src/os/unix/ngx_files.c index 1c82a8ead0c..2fec1ece102 100644 --- a/src/os/unix/ngx_files.c +++ b/src/os/unix/ngx_files.c @@ -110,6 +110,8 @@ ngx_thread_read(ngx_file_t *file, u_char *buf, size_t size, off_t offset, return NGX_ERROR; } + task->event.log = file->log; + file->thread_task = task; } @@ -493,6 +495,8 @@ ngx_thread_write_chain_to_file(ngx_file_t *file, ngx_chain_t *cl, off_t offset, return NGX_ERROR; } + task->event.log = file->log; + file->thread_task = task; } diff --git a/src/os/unix/ngx_linux_sendfile_chain.c b/src/os/unix/ngx_linux_sendfile_chain.c index 101d91a9a81..603f91722c3 100644 --- a/src/os/unix/ngx_linux_sendfile_chain.c +++ b/src/os/unix/ngx_linux_sendfile_chain.c @@ -332,6 +332,7 @@ ngx_linux_sendfile_thread(ngx_connection_t *c, ngx_buf_t *file, size_t size) return NGX_ERROR; } + task->event.log = c->log; task->handler = ngx_linux_sendfile_thread_handler; c->sendfile_task = task; From c251961c4186ce93cf6eb3c99bf5b7114535d490 Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Tue, 30 Jan 2024 03:20:05 +0300 Subject: [PATCH 087/279] Fixed request termination with AIO and subrequests (ticket #2555). When a request was terminated due to an error via ngx_http_terminate_request() while an AIO operation was running in a subrequest, various issues were observed. This happened because ngx_http_request_finalizer() was only set in the subrequest where ngx_http_terminate_request() was called, but not in the subrequest where the AIO operation was running. After completion of the AIO operation normal processing of the subrequest was resumed, leading to issues. In particular, in case of the upstream module, termination of the request called upstream cleanup, which closed the upstream connection. Attempts to further work with the upstream connection after AIO operation completion resulted in segfaults in ngx_ssl_recv(), "readv() failed (9: Bad file descriptor) while reading upstream" errors, or socket leaks. In ticket #2555, issues were observed with the following configuration with cache background update (with thread writing instrumented to introduce a delay, when a client closes the connection during an update): location = /background-and-aio-write { proxy_pass ... proxy_cache one; proxy_cache_valid 200 1s; proxy_cache_background_update on; proxy_cache_use_stale updating; aio threads; aio_write on; limit_rate 1000; } Similarly, the same issue can be seen with SSI, and can be caused by errors in subrequests, such as in the following configuration (where "/proxy" uses AIO, and "/sleep" returns 444 after some delay, causing request termination): location = /ssi-active-boom { ssi on; ssi_types *; return 200 ' '; limit_rate 1000; } Or the same with both AIO operation and the error in non-active subrequests (which needs slightly different handling, see below): location = /ssi-non-active-boom { ssi on; ssi_types *; return 200 ' '; limit_rate 1000; } Similarly, issues can be observed with just static files. However, with static files potential impact is limited due to timeout safeguards in ngx_http_writer(), and the fact that c->error is set during request termination. In a simple configuration with an AIO operation in the active subrequest, such as in the following configuration, the connection is closed right after completion of the AIO operation anyway, since ngx_http_writer() tries to write to the connection and fails due to c->error set: location = /ssi-active-static-boom { ssi on; ssi_types *; return 200 ' '; limit_rate 1000; } In the following configuration, with an AIO operation in a non-active subrequest, the connection is closed only after send_timeout expires: location = /ssi-non-active-static-boom { ssi on; ssi_types *; return 200 ' '; limit_rate 1000; } Fix is to introduce r->main->terminated flag, which is to be checked by AIO event handlers when the r->main->blocked counter is decremented. When the flag is set, handlers are expected to wake up the connection instead of the subrequest (which might be already cleaned up). Additionally, now ngx_http_request_finalizer() is always set in the active subrequest, so waking up the connection properly finalizes the request even if termination happened in a non-active subrequest. --- src/http/ngx_http_copy_filter_module.c | 19 ++++++-- src/http/ngx_http_file_cache.c | 63 ++++++++++++++++++++------ src/http/ngx_http_request.c | 5 ++ src/http/ngx_http_request.h | 1 + src/http/ngx_http_upstream.c | 6 +-- 5 files changed, 71 insertions(+), 23 deletions(-) diff --git a/src/http/ngx_http_copy_filter_module.c b/src/http/ngx_http_copy_filter_module.c index 6a96e003f2c..8e5de2bf701 100644 --- a/src/http/ngx_http_copy_filter_module.c +++ b/src/http/ngx_http_copy_filter_module.c @@ -208,9 +208,18 @@ ngx_http_copy_aio_event_handler(ngx_event_t *ev) r->main->blocked--; r->aio = 0; - r->write_event_handler(r); + if (r->main->terminated) { + /* + * trigger connection event handler if the request was + * terminated + */ + + c->write->handler(c->write); - ngx_http_run_posted_requests(c); + } else { + r->write_event_handler(r); + ngx_http_run_posted_requests(c); + } } #endif @@ -331,11 +340,11 @@ ngx_http_copy_thread_event_handler(ngx_event_t *ev) #endif - if (r->done) { + if (r->done || r->main->terminated) { /* * trigger connection event handler if the subrequest was - * already finalized; this can happen if the handler is used - * for sendfile() in threads + * already finalized (this can happen if the handler is used + * for sendfile() in threads), or if the request was terminated */ c->write->handler(c->write); diff --git a/src/http/ngx_http_file_cache.c b/src/http/ngx_http_file_cache.c index 6c49dd42df9..5209f003b0a 100644 --- a/src/http/ngx_http_file_cache.c +++ b/src/http/ngx_http_file_cache.c @@ -14,7 +14,7 @@ static ngx_int_t ngx_http_file_cache_lock(ngx_http_request_t *r, ngx_http_cache_t *c); static void ngx_http_file_cache_lock_wait_handler(ngx_event_t *ev); -static void ngx_http_file_cache_lock_wait(ngx_http_request_t *r, +static ngx_int_t ngx_http_file_cache_lock_wait(ngx_http_request_t *r, ngx_http_cache_t *c); static ngx_int_t ngx_http_file_cache_read(ngx_http_request_t *r, ngx_http_cache_t *c); @@ -463,6 +463,7 @@ ngx_http_file_cache_lock(ngx_http_request_t *r, ngx_http_cache_t *c) static void ngx_http_file_cache_lock_wait_handler(ngx_event_t *ev) { + ngx_int_t rc; ngx_connection_t *c; ngx_http_request_t *r; @@ -474,13 +475,31 @@ ngx_http_file_cache_lock_wait_handler(ngx_event_t *ev) ngx_log_debug2(NGX_LOG_DEBUG_HTTP, c->log, 0, "http file cache wait: \"%V?%V\"", &r->uri, &r->args); - ngx_http_file_cache_lock_wait(r, r->cache); + rc = ngx_http_file_cache_lock_wait(r, r->cache); - ngx_http_run_posted_requests(c); + if (rc == NGX_AGAIN) { + return; + } + + r->cache->waiting = 0; + r->main->blocked--; + + if (r->main->terminated) { + /* + * trigger connection event handler if the request was + * terminated + */ + + c->write->handler(c->write); + + } else { + r->write_event_handler(r); + ngx_http_run_posted_requests(c); + } } -static void +static ngx_int_t ngx_http_file_cache_lock_wait(ngx_http_request_t *r, ngx_http_cache_t *c) { ngx_uint_t wait; @@ -495,7 +514,7 @@ ngx_http_file_cache_lock_wait(ngx_http_request_t *r, ngx_http_cache_t *c) ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, "cache lock timeout"); c->lock_timeout = 0; - goto wakeup; + return NGX_OK; } cache = c->file_cache; @@ -513,14 +532,10 @@ ngx_http_file_cache_lock_wait(ngx_http_request_t *r, ngx_http_cache_t *c) if (wait) { ngx_add_timer(&c->wait_event, (timer > 500) ? 500 : timer); - return; + return NGX_AGAIN; } -wakeup: - - c->waiting = 0; - r->main->blocked--; - r->write_event_handler(r); + return NGX_OK; } @@ -753,9 +768,18 @@ ngx_http_cache_aio_event_handler(ngx_event_t *ev) r->main->blocked--; r->aio = 0; - r->write_event_handler(r); + if (r->main->terminated) { + /* + * trigger connection event handler if the request was + * terminated + */ + + c->write->handler(c->write); - ngx_http_run_posted_requests(c); + } else { + r->write_event_handler(r); + ngx_http_run_posted_requests(c); + } } #endif @@ -836,9 +860,18 @@ ngx_http_cache_thread_event_handler(ngx_event_t *ev) r->main->blocked--; r->aio = 0; - r->write_event_handler(r); + if (r->main->terminated) { + /* + * trigger connection event handler if the request was + * terminated + */ - ngx_http_run_posted_requests(c); + c->write->handler(c->write); + + } else { + r->write_event_handler(r); + ngx_http_run_posted_requests(c); + } } #endif diff --git a/src/http/ngx_http_request.c b/src/http/ngx_http_request.c index 21738ff4c58..3cca57cf5ee 100644 --- a/src/http/ngx_http_request.c +++ b/src/http/ngx_http_request.c @@ -2694,6 +2694,8 @@ ngx_http_terminate_request(ngx_http_request_t *r, ngx_int_t rc) ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "http terminate request count:%d", mr->count); + mr->terminated = 1; + if (rc > 0 && (mr->headers_out.status == 0 || mr->connection->sent == 0)) { mr->headers_out.status = rc; } @@ -2716,8 +2718,11 @@ ngx_http_terminate_request(ngx_http_request_t *r, ngx_int_t rc) if (mr->write_event_handler) { if (mr->blocked) { + r = r->connection->data; + r->connection->error = 1; r->write_event_handler = ngx_http_request_finalizer; + return; } diff --git a/src/http/ngx_http_request.h b/src/http/ngx_http_request.h index 48b052bbc56..65c8333f8eb 100644 --- a/src/http/ngx_http_request.h +++ b/src/http/ngx_http_request.h @@ -550,6 +550,7 @@ struct ngx_http_request_s { unsigned root_tested:1; unsigned done:1; unsigned logged:1; + unsigned terminated:1; unsigned buffered:4; diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c index 99215ad8270..a67d5043211 100644 --- a/src/http/ngx_http_upstream.c +++ b/src/http/ngx_http_upstream.c @@ -3997,11 +3997,11 @@ ngx_http_upstream_thread_event_handler(ngx_event_t *ev) #endif - if (r->done) { + if (r->done || r->main->terminated) { /* * trigger connection event handler if the subrequest was - * already finalized; this can happen if the handler is used - * for sendfile() in threads + * already finalized (this can happen if the handler is used + * for sendfile() in threads), or if the request was terminated */ c->write->handler(c->write); From 6f2059147f20d1bd2cd6ff01ea71bf31ec9c2845 Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Tue, 30 Jan 2024 03:20:10 +0300 Subject: [PATCH 088/279] Upstream: fixed usage of closed sockets with filter finalization. When filter finalization is triggered when working with an upstream server, and error_page redirects request processing to some simple handler, ngx_http_request_finalize() triggers request termination when the response is sent. In particular, via the upstream cleanup handler, nginx will close the upstream connection and the corresponding socket. Still, this can happen to be with ngx_event_pipe() on stack. While the code will set p->downstream_error due to NGX_ERROR returned from the output filter chain by filter finalization, otherwise the error will be ignored till control returns to ngx_http_upstream_process_request(). And event pipe might try reading from the (already closed) socket, resulting in "readv() failed (9: Bad file descriptor) while reading upstream" errors (or even segfaults with SSL). Such errors were seen with the following configuration: location /t2 { proxy_pass http://127.0.0.1:8080/big; image_filter_buffer 10m; image_filter resize 150 100; error_page 415 = /empty; } location /empty { return 204; } location /big { # big enough static file } Fix is to clear p->upstream in ngx_http_upstream_finalize_request(), and ensure that p->upstream is checked in ngx_event_pipe_read_upstream() and when handling events at ngx_event_pipe() exit. --- src/event/ngx_event_pipe.c | 8 ++++++-- src/http/ngx_http_upstream.c | 4 ++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/event/ngx_event_pipe.c b/src/event/ngx_event_pipe.c index 54412e130f6..d774903ec6e 100644 --- a/src/event/ngx_event_pipe.c +++ b/src/event/ngx_event_pipe.c @@ -57,7 +57,9 @@ ngx_event_pipe(ngx_event_pipe_t *p, ngx_int_t do_write) do_write = 1; } - if (p->upstream->fd != (ngx_socket_t) -1) { + if (p->upstream + && p->upstream->fd != (ngx_socket_t) -1) + { rev = p->upstream->read; flags = (rev->eof || rev->error) ? NGX_CLOSE_EVENT : 0; @@ -108,7 +110,9 @@ ngx_event_pipe_read_upstream(ngx_event_pipe_t *p) ngx_msec_t delay; ngx_chain_t *chain, *cl, *ln; - if (p->upstream_eof || p->upstream_error || p->upstream_done) { + if (p->upstream_eof || p->upstream_error || p->upstream_done + || p->upstream == NULL) + { return NGX_OK; } diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c index a67d5043211..2ce9f211441 100644 --- a/src/http/ngx_http_upstream.c +++ b/src/http/ngx_http_upstream.c @@ -4574,6 +4574,10 @@ ngx_http_upstream_finalize_request(ngx_http_request_t *r, u->peer.connection = NULL; + if (u->pipe) { + u->pipe->upstream = NULL; + } + if (u->pipe && u->pipe->temp_file) { ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "http upstream temp fd: %d", From 771cf15704b68e39756f33a23762be5da0e816ac Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Tue, 30 Jan 2024 19:14:16 +0400 Subject: [PATCH 089/279] Year 2024. --- docs/text/LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/text/LICENSE b/docs/text/LICENSE index 2792c4498c5..985470ef99c 100644 --- a/docs/text/LICENSE +++ b/docs/text/LICENSE @@ -1,6 +1,6 @@ /* * Copyright (C) 2002-2021 Igor Sysoev - * Copyright (C) 2011-2023 Nginx, Inc. + * Copyright (C) 2011-2024 Nginx, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without From 2a10e48620d430bc2d44f36249c33fb1813aa507 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Tue, 30 Jan 2024 19:18:31 +0400 Subject: [PATCH 090/279] SSL: fixed $ssl_curves allocation error handling. --- src/event/ngx_event_openssl.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c index 104dd03f274..89f277fe56a 100644 --- a/src/event/ngx_event_openssl.c +++ b/src/event/ngx_event_openssl.c @@ -5187,6 +5187,9 @@ ngx_ssl_get_curves(ngx_connection_t *c, ngx_pool_t *pool, ngx_str_t *s) } curves = ngx_palloc(pool, n * sizeof(int)); + if (curves == NULL) { + return NGX_ERROR; + } n = SSL_get1_curves(c->ssl->connection, curves); len = 0; From 71a0a4acdbb9ed0a8ef269a28218365cde00415d Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Tue, 30 Jan 2024 19:19:26 +0400 Subject: [PATCH 091/279] HTTP/3: added more compatibility checks for "listen ... quic". Now "fastopen", "backlog", "accept_filter", "deferred", and "so_keepalive" parameters are not allowed with "quic" in the "listen" directive. Reported by Izorkin. --- src/http/ngx_http_core_module.c | 36 ++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/src/http/ngx_http_core_module.c b/src/http/ngx_http_core_module.c index 97a91aee266..033a3bf6403 100644 --- a/src/http/ngx_http_core_module.c +++ b/src/http/ngx_http_core_module.c @@ -3961,7 +3961,7 @@ ngx_http_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) ngx_str_t *value, size; ngx_url_t u; - ngx_uint_t n, i; + ngx_uint_t n, i, backlog; ngx_http_listen_opt_t lsopt; cscf->listen = 1; @@ -4000,6 +4000,8 @@ ngx_http_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) lsopt.ipv6only = 1; #endif + backlog = 0; + for (n = 2; n < cf->args->nelts; n++) { if (ngx_strcmp(value[n].data, "default_server") == 0 @@ -4058,6 +4060,8 @@ ngx_http_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) return NGX_CONF_ERROR; } + backlog = 1; + continue; } @@ -4305,9 +4309,29 @@ ngx_http_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) return NGX_CONF_ERROR; } -#if (NGX_HTTP_V3) - if (lsopt.quic) { +#if (NGX_HAVE_TCP_FASTOPEN) + if (lsopt.fastopen != -1) { + return "\"fastopen\" parameter is incompatible with \"quic\""; + } +#endif + + if (backlog) { + return "\"backlog\" parameter is incompatible with \"quic\""; + } + +#if (NGX_HAVE_DEFERRED_ACCEPT && defined SO_ACCEPTFILTER) + if (lsopt.accept_filter) { + return "\"accept_filter\" parameter is incompatible with \"quic\""; + } +#endif + +#if (NGX_HAVE_DEFERRED_ACCEPT && defined TCP_DEFER_ACCEPT) + if (lsopt.deferred_accept) { + return "\"deferred\" parameter is incompatible with \"quic\""; + } +#endif + #if (NGX_HTTP_SSL) if (lsopt.ssl) { return "\"ssl\" parameter is incompatible with \"quic\""; @@ -4320,13 +4344,15 @@ ngx_http_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) } #endif + if (lsopt.so_keepalive) { + return "\"so_keepalive\" parameter is incompatible with \"quic\""; + } + if (lsopt.proxy_protocol) { return "\"proxy_protocol\" parameter is incompatible with \"quic\""; } } -#endif - for (n = 0; n < u.naddrs; n++) { for (i = 0; i < n; i++) { From ed47f72a85fb6279e2ba5d431f64ea4db695cf4e Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Wed, 14 Feb 2024 16:56:28 +0400 Subject: [PATCH 092/279] QUIC: fixed unsent MTU probe acknowledgement. Previously if an MTU probe send failed early in ngx_quic_frame_sendto() due to allocation error or congestion control, the application level packet number was not increased, but was still saved as MTU probe packet number. Later when a packet with this number was acknowledged, the unsent MTU probe was acknowledged as well. This could result in discovering a bigger MTU than supported by the path, which could lead to EMSGSIZE (Message too long) errors while sending further packets. The problem existed since PMTUD was introduced in 58afcd72446f (1.25.2). Back then only the unlikely memory allocation error could trigger it. However in efcdaa66df2e congestion control was added to ngx_quic_frame_sendto() which can now trigger the issue with a higher probability. --- src/event/quic/ngx_event_quic_migration.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/event/quic/ngx_event_quic_migration.c b/src/event/quic/ngx_event_quic_migration.c index 3c1bbaf43f0..2d1467e1493 100644 --- a/src/event/quic/ngx_event_quic_migration.c +++ b/src/event/quic/ngx_event_quic_migration.c @@ -909,6 +909,7 @@ static ngx_int_t ngx_quic_send_path_mtu_probe(ngx_connection_t *c, ngx_quic_path_t *path) { size_t mtu; + uint64_t pnum; ngx_int_t rc; ngx_uint_t log_error; ngx_quic_frame_t *frame; @@ -925,7 +926,7 @@ ngx_quic_send_path_mtu_probe(ngx_connection_t *c, ngx_quic_path_t *path) qc = ngx_quic_get_connection(c); ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_application); - path->mtu_pnum[path->tries] = ctx->pnum; + pnum = ctx->pnum; ngx_log_debug4(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic path seq:%uL send probe " @@ -943,14 +944,18 @@ ngx_quic_send_path_mtu_probe(ngx_connection_t *c, ngx_quic_path_t *path) path->mtu = mtu; c->log_error = log_error; + if (rc == NGX_OK) { + path->mtu_pnum[path->tries] = pnum; + return NGX_OK; + } + + ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic path seq:%uL rejected mtu:%uz", + path->seqnum, path->mtud); + if (rc == NGX_ERROR) { if (c->write->error) { c->write->error = 0; - - ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, - "quic path seq:%uL rejected mtu:%uz", - path->seqnum, path->mtud); - return NGX_DECLINED; } @@ -976,7 +981,7 @@ ngx_quic_handle_path_mtu(ngx_connection_t *c, ngx_quic_path_t *path, pnum = path->mtu_pnum[i]; if (pnum == NGX_QUIC_UNSET_PN) { - break; + continue; } if (pnum < min || pnum > max) { From 5902baf680609f884a1e11ff2b82a0bffb3724cc Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Wed, 14 Feb 2024 15:55:34 +0400 Subject: [PATCH 093/279] QUIC: trial packet decryption in response to invalid key update. Inspired by RFC 9001, Section 6.3, trial packet decryption with the current keys is now used to avoid a timing side-channel signal. Further, this fixes segfault while accessing missing next keys (ticket #2585). --- src/event/quic/ngx_event_quic_protection.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/event/quic/ngx_event_quic_protection.c b/src/event/quic/ngx_event_quic_protection.c index 88e6954cffb..8223626b65f 100644 --- a/src/event/quic/ngx_event_quic_protection.c +++ b/src/event/quic/ngx_event_quic_protection.c @@ -1144,8 +1144,19 @@ ngx_quic_decrypt(ngx_quic_header_t *pkt, uint64_t *largest_pn) key_phase = (pkt->flags & NGX_QUIC_PKT_KPHASE) != 0; if (key_phase != pkt->key_phase) { - secret = &pkt->keys->next_key.client; - pkt->key_update = 1; + if (pkt->keys->next_key.client.ctx != NULL) { + secret = &pkt->keys->next_key.client; + pkt->key_update = 1; + + } else { + /* + * RFC 9001, 6.3. Timing of Receive Key Generation. + * + * Trial decryption to avoid timing side-channel. + */ + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, pkt->log, 0, + "quic next key missing"); + } } } From 5818f8a6693b3c0d95021f2ee58b69dcf848911c Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Wed, 14 Feb 2024 15:55:37 +0400 Subject: [PATCH 094/279] QUIC: fixed stream cleanup (ticket #2586). Stream connection cleanup handler ngx_quic_stream_cleanup_handler() calls ngx_quic_shutdown_stream() after which it resets the pointer from quic stream to the connection (sc->connection = NULL). Previously if this call failed, sc->connection retained the old value, while the connection was freed by the application code. This resulted later in a second attempt to close the freed connection, which lead to allocator double free error. The fix is to reset the sc->connection pointer in case of error. --- src/event/quic/ngx_event_quic_streams.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/event/quic/ngx_event_quic_streams.c b/src/event/quic/ngx_event_quic_streams.c index df04d0f0740..178b805e450 100644 --- a/src/event/quic/ngx_event_quic_streams.c +++ b/src/event/quic/ngx_event_quic_streams.c @@ -1097,6 +1097,7 @@ ngx_quic_stream_cleanup_handler(void *data) "quic stream id:0x%xL cleanup", qs->id); if (ngx_quic_shutdown_stream(c, NGX_RDWR_SHUTDOWN) != NGX_OK) { + qs->connection = NULL; goto failed; } From 4bef3c3367e44488d2b26dd3683e5121b2424d28 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Wed, 14 Feb 2024 15:55:42 +0400 Subject: [PATCH 095/279] Updated OpenSSL and zlib used for win32 builds. --- misc/GNUmakefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misc/GNUmakefile b/misc/GNUmakefile index 067dd7481d9..dca0e263750 100644 --- a/misc/GNUmakefile +++ b/misc/GNUmakefile @@ -6,8 +6,8 @@ TEMP = tmp CC = cl OBJS = objs.msvc8 -OPENSSL = openssl-3.0.11 -ZLIB = zlib-1.3 +OPENSSL = openssl-3.0.13 +ZLIB = zlib-1.3.1 PCRE = pcre2-10.39 From c8b288cd8a25628a167a874261069302f3cd7548 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Wed, 14 Feb 2024 15:55:46 +0400 Subject: [PATCH 096/279] nginx-1.25.4-RELEASE --- docs/xml/nginx/changes.xml | 76 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/docs/xml/nginx/changes.xml b/docs/xml/nginx/changes.xml index d2d2caccb65..6d11d6da406 100644 --- a/docs/xml/nginx/changes.xml +++ b/docs/xml/nginx/changes.xml @@ -5,6 +5,82 @@ + + + + +при использовании HTTP/3 в рабочем процессе мог произойти segmentation fault +во время обработки специально созданной QUIC-сессии +(CVE-2024-24989, CVE-2024-24990). + + +when using HTTP/3 a segmentation fault might occur in a worker process +while processing a specially crafted QUIC session +(CVE-2024-24989, CVE-2024-24990). + + + + + +соединения с незавершенными AIO-операциями могли закрываться преждевременно +во время плавного завершения старых рабочих процессов. + + +connections with pending AIO operations might be closed prematurely +during graceful shutdown of old worker processes. + + + + + +теперь nginx не пишет в лог сообщения об утечке сокетов, +если во время плавного завершения старых рабочих процессов +было запрошено быстрое завершение. + + +socket leak alerts no longer logged when fast shutdown +was requested after graceful shutdown of old worker processes. + + + + + +при использовании AIO в подзапросе могла происходить +ошибка на сокете, утечка сокетов, +либо segmentation fault в рабочем процессе (при SSL-проксировании). + + +a socket descriptor error, a socket leak, +or a segmentation fault in a worker process (for SSL proxying) +might occur if AIO was used in a subrequest. + + + + + +в рабочем процессе мог произойти segmentation fault, +если использовалось SSL-проксирование и директива image_filter, +а ошибки с кодом 415 перенаправлялись с помощью директивы error_page. + + +a segmentation fault might occur in a worker process +if SSL proxying was used along with the "image_filter" directive +and errors with code 415 were redirected with the "error_page" directive. + + + + + +Исправления и улучшения в HTTP/3. + + +Bugfixes and improvements in HTTP/3. + + + + + + From ef96f5835468ff8d40df29b0ddbc04ec1e5e1582 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Wed, 14 Feb 2024 20:03:00 +0400 Subject: [PATCH 097/279] release-1.25.4 tag --- .hgtags | 1 + 1 file changed, 1 insertion(+) diff --git a/.hgtags b/.hgtags index 58d85238431..387b38397f0 100644 --- a/.hgtags +++ b/.hgtags @@ -476,3 +476,4 @@ ac779115ed6ee4f3039e9aea414a54e560450ee2 release-1.23.4 f8134640e8615448205785cf00b0bc810489b495 release-1.25.1 1d839f05409d1a50d0f15a2bf36547001f99ae40 release-1.25.2 294a3d07234f8f65d7b0e0b0e2c5b05c12c5da0a release-1.25.3 +173a0a7dbce569adbb70257c6ec4f0f6bc585009 release-1.25.4 From b1a2b3ebdf336a422ceb94517a0943e28688831d Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Thu, 21 Mar 2024 17:06:21 +0400 Subject: [PATCH 098/279] Version bump. --- src/core/nginx.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/nginx.h b/src/core/nginx.h index dc1308fd13a..f18c3930fc5 100644 --- a/src/core/nginx.h +++ b/src/core/nginx.h @@ -9,8 +9,8 @@ #define _NGINX_H_INCLUDED_ -#define nginx_version 1025004 -#define NGINX_VERSION "1.25.4" +#define nginx_version 1025005 +#define NGINX_VERSION "1.25.5" #define NGINX_VER "nginx/" NGINX_VERSION #ifdef NGX_BUILD From 72e5d6ac19a93c9fb64678dd33ea185757a6021a Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Wed, 13 Dec 2023 18:04:55 +0400 Subject: [PATCH 099/279] Stream: socket peek in preread phase. Previously, preread buffer was always read out from socket, which made it impossible to terminate SSL on the connection without introducing additional SSL BIOs. The following patches will rely on this. Now, when possible, recv(MSG_PEEK) is used instead, which keeps data in socket. It's called if SSL is not already terminated and if an egde-triggered event method is used. For epoll, EPOLLRDHUP support is also required. --- src/stream/ngx_stream_core_module.c | 189 +++++++++++++++++++++------- 1 file changed, 146 insertions(+), 43 deletions(-) diff --git a/src/stream/ngx_stream_core_module.c b/src/stream/ngx_stream_core_module.c index f0b79341d85..2ef39af2425 100644 --- a/src/stream/ngx_stream_core_module.c +++ b/src/stream/ngx_stream_core_module.c @@ -10,6 +10,11 @@ #include +static ngx_uint_t ngx_stream_preread_can_peek(ngx_connection_t *c); +static ngx_int_t ngx_stream_preread_peek(ngx_stream_session_t *s, + ngx_stream_phase_handler_t *ph); +static ngx_int_t ngx_stream_preread(ngx_stream_session_t *s, + ngx_stream_phase_handler_t *ph); static ngx_int_t ngx_stream_core_preconfiguration(ngx_conf_t *cf); static void *ngx_stream_core_create_main_conf(ngx_conf_t *cf); static char *ngx_stream_core_init_main_conf(ngx_conf_t *cf, void *conf); @@ -203,8 +208,6 @@ ngx_int_t ngx_stream_core_preread_phase(ngx_stream_session_t *s, ngx_stream_phase_handler_t *ph) { - size_t size; - ssize_t n; ngx_int_t rc; ngx_connection_t *c; ngx_stream_core_srv_conf_t *cscf; @@ -217,57 +220,34 @@ ngx_stream_core_preread_phase(ngx_stream_session_t *s, if (c->read->timedout) { rc = NGX_STREAM_OK; - - } else if (c->read->timer_set) { - rc = NGX_AGAIN; - - } else { - rc = ph->handler(s); + goto done; } - while (rc == NGX_AGAIN) { - - if (c->buffer == NULL) { - c->buffer = ngx_create_temp_buf(c->pool, cscf->preread_buffer_size); - if (c->buffer == NULL) { - rc = NGX_ERROR; - break; - } - } - - size = c->buffer->end - c->buffer->last; - - if (size == 0) { - ngx_log_error(NGX_LOG_ERR, c->log, 0, "preread buffer full"); - rc = NGX_STREAM_BAD_REQUEST; - break; - } - - if (c->read->eof) { - rc = NGX_STREAM_OK; - break; - } - - if (!c->read->ready) { - break; - } - - n = c->recv(c, c->buffer->last, size); + if (!c->read->timer_set) { + rc = ph->handler(s); - if (n == NGX_ERROR || n == 0) { - rc = NGX_STREAM_OK; - break; + if (rc != NGX_AGAIN) { + goto done; } + } - if (n == NGX_AGAIN) { - break; + if (c->buffer == NULL) { + c->buffer = ngx_create_temp_buf(c->pool, cscf->preread_buffer_size); + if (c->buffer == NULL) { + rc = NGX_ERROR; + goto done; } + } - c->buffer->last += n; + if (ngx_stream_preread_can_peek(c)) { + rc = ngx_stream_preread_peek(s, ph); - rc = ph->handler(s); + } else { + rc = ngx_stream_preread(s, ph); } +done: + if (rc == NGX_AGAIN) { if (ngx_handle_read_event(c->read, 0) != NGX_OK) { ngx_stream_finalize_session(s, NGX_STREAM_INTERNAL_SERVER_ERROR); @@ -311,6 +291,129 @@ ngx_stream_core_preread_phase(ngx_stream_session_t *s, } +static ngx_uint_t +ngx_stream_preread_can_peek(ngx_connection_t *c) +{ +#if (NGX_STREAM_SSL) + if (c->ssl) { + return 0; + } +#endif + + if ((ngx_event_flags & NGX_USE_CLEAR_EVENT) == 0) { + return 0; + } + +#if (NGX_HAVE_KQUEUE) + if (ngx_event_flags & NGX_USE_KQUEUE_EVENT) { + return 1; + } +#endif + +#if (NGX_HAVE_EPOLLRDHUP) + if ((ngx_event_flags & NGX_USE_EPOLL_EVENT) && ngx_use_epoll_rdhup) { + return 1; + } +#endif + + return 0; +} + + +static ngx_int_t +ngx_stream_preread_peek(ngx_stream_session_t *s, ngx_stream_phase_handler_t *ph) +{ + ssize_t n; + ngx_int_t rc; + ngx_err_t err; + ngx_connection_t *c; + + c = s->connection; + + n = recv(c->fd, (char *) c->buffer->last, + c->buffer->end - c->buffer->last, MSG_PEEK); + + err = ngx_socket_errno; + + ngx_log_debug1(NGX_LOG_DEBUG_STREAM, c->log, 0, "stream recv(): %z", n); + + if (n == -1) { + if (err == NGX_EAGAIN) { + c->read->ready = 0; + return NGX_AGAIN; + } + + ngx_connection_error(c, err, "recv() failed"); + return NGX_STREAM_OK; + } + + if (n == 0) { + return NGX_STREAM_OK; + } + + c->buffer->last += n; + + rc = ph->handler(s); + + if (rc != NGX_AGAIN) { + c->buffer->last = c->buffer->pos; + return rc; + } + + if (c->buffer->last == c->buffer->end) { + ngx_log_error(NGX_LOG_ERR, c->log, 0, "preread buffer full"); + return NGX_STREAM_BAD_REQUEST; + } + + if (c->read->pending_eof) { + return NGX_STREAM_OK; + } + + c->buffer->last = c->buffer->pos; + + return NGX_AGAIN; +} + + +static ngx_int_t +ngx_stream_preread(ngx_stream_session_t *s, ngx_stream_phase_handler_t *ph) +{ + ssize_t n; + ngx_int_t rc; + ngx_connection_t *c; + + c = s->connection; + + while (c->read->ready) { + + n = c->recv(c, c->buffer->last, c->buffer->end - c->buffer->last); + + if (n == NGX_AGAIN) { + return NGX_AGAIN; + } + + if (n == NGX_ERROR || n == 0) { + return NGX_STREAM_OK; + } + + c->buffer->last += n; + + rc = ph->handler(s); + + if (rc != NGX_AGAIN) { + return rc; + } + + if (c->buffer->last == c->buffer->end) { + ngx_log_error(NGX_LOG_ERR, c->log, 0, "preread buffer full"); + return NGX_STREAM_BAD_REQUEST; + } + } + + return NGX_AGAIN; +} + + ngx_int_t ngx_stream_core_content_phase(ngx_stream_session_t *s, ngx_stream_phase_handler_t *ph) From d21675228a0ba8d4331e05c60660228a5d3326de Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Thu, 14 Dec 2023 21:58:39 +0400 Subject: [PATCH 100/279] Stream: virtual servers. Server name is taken either from ngx_stream_ssl_module or ngx_stream_ssl_preread_module. The change adds "default_server" parameter to the "listen" directive, as well as the following directives: "server_names_hash_max_size", "server_names_hash_bucket_size", "server_name" and "ssl_reject_handshake". --- src/stream/ngx_stream.c | 834 +++++++++++++++++---- src/stream/ngx_stream.h | 160 ++-- src/stream/ngx_stream_core_module.c | 480 +++++++++--- src/stream/ngx_stream_handler.c | 10 +- src/stream/ngx_stream_ssl_module.c | 224 +++++- src/stream/ngx_stream_ssl_module.h | 5 +- src/stream/ngx_stream_ssl_preread_module.c | 57 +- 7 files changed, 1416 insertions(+), 354 deletions(-) diff --git a/src/stream/ngx_stream.c b/src/stream/ngx_stream.c index 3304c843cef..9e16cd382f1 100644 --- a/src/stream/ngx_stream.c +++ b/src/stream/ngx_stream.c @@ -16,16 +16,34 @@ static ngx_int_t ngx_stream_init_phases(ngx_conf_t *cf, ngx_stream_core_main_conf_t *cmcf); static ngx_int_t ngx_stream_init_phase_handlers(ngx_conf_t *cf, ngx_stream_core_main_conf_t *cmcf); -static ngx_int_t ngx_stream_add_ports(ngx_conf_t *cf, ngx_array_t *ports, - ngx_stream_listen_t *listen); -static char *ngx_stream_optimize_servers(ngx_conf_t *cf, ngx_array_t *ports); + +static ngx_int_t ngx_stream_add_addresses(ngx_conf_t *cf, + ngx_stream_core_srv_conf_t *cscf, ngx_stream_conf_port_t *port, + ngx_stream_listen_opt_t *lsopt); +static ngx_int_t ngx_stream_add_address(ngx_conf_t *cf, + ngx_stream_core_srv_conf_t *cscf, ngx_stream_conf_port_t *port, + ngx_stream_listen_opt_t *lsopt); +static ngx_int_t ngx_stream_add_server(ngx_conf_t *cf, + ngx_stream_core_srv_conf_t *cscf, ngx_stream_conf_addr_t *addr); + +static ngx_int_t ngx_stream_optimize_servers(ngx_conf_t *cf, + ngx_stream_core_main_conf_t *cmcf, ngx_array_t *ports); +static ngx_int_t ngx_stream_server_names(ngx_conf_t *cf, + ngx_stream_core_main_conf_t *cmcf, ngx_stream_conf_addr_t *addr); +static ngx_int_t ngx_stream_cmp_conf_addrs(const void *one, const void *two); +static int ngx_libc_cdecl ngx_stream_cmp_dns_wildcards(const void *one, + const void *two); + +static ngx_int_t ngx_stream_init_listening(ngx_conf_t *cf, + ngx_stream_conf_port_t *port); +static ngx_listening_t *ngx_stream_add_listening(ngx_conf_t *cf, + ngx_stream_conf_addr_t *addr); static ngx_int_t ngx_stream_add_addrs(ngx_conf_t *cf, ngx_stream_port_t *stport, ngx_stream_conf_addr_t *addr); #if (NGX_HAVE_INET6) static ngx_int_t ngx_stream_add_addrs6(ngx_conf_t *cf, ngx_stream_port_t *stport, ngx_stream_conf_addr_t *addr); #endif -static ngx_int_t ngx_stream_cmp_conf_addrs(const void *one, const void *two); ngx_uint_t ngx_stream_max_module; @@ -74,10 +92,8 @@ static char * ngx_stream_block(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) { char *rv; - ngx_uint_t i, m, mi, s; + ngx_uint_t mi, m, s; ngx_conf_t pcf; - ngx_array_t ports; - ngx_stream_listen_t *listen; ngx_stream_module_t *module; ngx_stream_conf_ctx_t *ctx; ngx_stream_core_srv_conf_t **cscfp; @@ -251,21 +267,13 @@ ngx_stream_block(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) return NGX_CONF_ERROR; } - if (ngx_array_init(&ports, cf->temp_pool, 4, sizeof(ngx_stream_conf_port_t)) - != NGX_OK) - { - return NGX_CONF_ERROR; - } + /* optimize the lists of ports, addresses and server names */ - listen = cmcf->listen.elts; - - for (i = 0; i < cmcf->listen.nelts; i++) { - if (ngx_stream_add_ports(cf, &ports, &listen[i]) != NGX_OK) { - return NGX_CONF_ERROR; - } + if (ngx_stream_optimize_servers(cf, cmcf, cmcf->ports) != NGX_OK) { + return NGX_CONF_ERROR; } - return ngx_stream_optimize_servers(cf, &ports); + return NGX_CONF_OK; } @@ -377,73 +385,295 @@ ngx_stream_init_phase_handlers(ngx_conf_t *cf, } -static ngx_int_t -ngx_stream_add_ports(ngx_conf_t *cf, ngx_array_t *ports, - ngx_stream_listen_t *listen) +ngx_int_t +ngx_stream_add_listen(ngx_conf_t *cf, ngx_stream_core_srv_conf_t *cscf, + ngx_stream_listen_opt_t *lsopt) { - in_port_t p; - ngx_uint_t i; - struct sockaddr *sa; - ngx_stream_conf_port_t *port; - ngx_stream_conf_addr_t *addr; + in_port_t p; + ngx_uint_t i; + struct sockaddr *sa; + ngx_stream_conf_port_t *port; + ngx_stream_core_main_conf_t *cmcf; + + cmcf = ngx_stream_conf_get_module_main_conf(cf, ngx_stream_core_module); + + if (cmcf->ports == NULL) { + cmcf->ports = ngx_array_create(cf->temp_pool, 2, + sizeof(ngx_stream_conf_port_t)); + if (cmcf->ports == NULL) { + return NGX_ERROR; + } + } - sa = listen->sockaddr; + sa = lsopt->sockaddr; p = ngx_inet_get_port(sa); - port = ports->elts; - for (i = 0; i < ports->nelts; i++) { + port = cmcf->ports->elts; + for (i = 0; i < cmcf->ports->nelts; i++) { - if (p == port[i].port - && listen->type == port[i].type - && sa->sa_family == port[i].family) + if (p != port[i].port + || lsopt->type != port[i].type + || sa->sa_family != port[i].family) { - /* a port is already in the port list */ - - port = &port[i]; - goto found; + continue; } + + /* a port is already in the port list */ + + return ngx_stream_add_addresses(cf, cscf, &port[i], lsopt); } /* add a port to the port list */ - port = ngx_array_push(ports); + port = ngx_array_push(cmcf->ports); if (port == NULL) { return NGX_ERROR; } port->family = sa->sa_family; - port->type = listen->type; + port->type = lsopt->type; port->port = p; + port->addrs.elts = NULL; - if (ngx_array_init(&port->addrs, cf->temp_pool, 2, - sizeof(ngx_stream_conf_addr_t)) - != NGX_OK) - { - return NGX_ERROR; + return ngx_stream_add_address(cf, cscf, port, lsopt); +} + + +static ngx_int_t +ngx_stream_add_addresses(ngx_conf_t *cf, ngx_stream_core_srv_conf_t *cscf, + ngx_stream_conf_port_t *port, ngx_stream_listen_opt_t *lsopt) +{ + ngx_uint_t i, default_server, proxy_protocol, + protocols, protocols_prev; + ngx_stream_conf_addr_t *addr; +#if (NGX_STREAM_SSL) + ngx_uint_t ssl; +#endif + + /* + * we cannot compare whole sockaddr struct's as kernel + * may fill some fields in inherited sockaddr struct's + */ + + addr = port->addrs.elts; + + for (i = 0; i < port->addrs.nelts; i++) { + + if (ngx_cmp_sockaddr(lsopt->sockaddr, lsopt->socklen, + addr[i].opt.sockaddr, + addr[i].opt.socklen, 0) + != NGX_OK) + { + continue; + } + + /* the address is already in the address list */ + + if (ngx_stream_add_server(cf, cscf, &addr[i]) != NGX_OK) { + return NGX_ERROR; + } + + /* preserve default_server bit during listen options overwriting */ + default_server = addr[i].opt.default_server; + + proxy_protocol = lsopt->proxy_protocol || addr[i].opt.proxy_protocol; + protocols = lsopt->proxy_protocol; + protocols_prev = addr[i].opt.proxy_protocol; + +#if (NGX_STREAM_SSL) + ssl = lsopt->ssl || addr[i].opt.ssl; + protocols |= lsopt->ssl << 1; + protocols_prev |= addr[i].opt.ssl << 1; +#endif + + if (lsopt->set) { + + if (addr[i].opt.set) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "duplicate listen options for %V", + &addr[i].opt.addr_text); + return NGX_ERROR; + } + + addr[i].opt = *lsopt; + } + + /* check the duplicate "default" server for this address:port */ + + if (lsopt->default_server) { + + if (default_server) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "a duplicate default server for %V", + &addr[i].opt.addr_text); + return NGX_ERROR; + } + + default_server = 1; + addr[i].default_server = cscf; + } + + /* check for conflicting protocol options */ + + if ((protocols | protocols_prev) != protocols_prev) { + + /* options added */ + + if ((addr[i].opt.set && !lsopt->set) + || addr[i].protocols_changed + || (protocols | protocols_prev) != protocols) + { + ngx_conf_log_error(NGX_LOG_WARN, cf, 0, + "protocol options redefined for %V", + &addr[i].opt.addr_text); + } + + addr[i].protocols = protocols_prev; + addr[i].protocols_set = 1; + addr[i].protocols_changed = 1; + + } else if ((protocols_prev | protocols) != protocols) { + + /* options removed */ + + if (lsopt->set + || (addr[i].protocols_set && protocols != addr[i].protocols)) + { + ngx_conf_log_error(NGX_LOG_WARN, cf, 0, + "protocol options redefined for %V", + &addr[i].opt.addr_text); + } + + addr[i].protocols = protocols; + addr[i].protocols_set = 1; + addr[i].protocols_changed = 1; + + } else { + + /* the same options */ + + if ((lsopt->set && addr[i].protocols_changed) + || (addr[i].protocols_set && protocols != addr[i].protocols)) + { + ngx_conf_log_error(NGX_LOG_WARN, cf, 0, + "protocol options redefined for %V", + &addr[i].opt.addr_text); + } + + addr[i].protocols = protocols; + addr[i].protocols_set = 1; + } + + addr[i].opt.default_server = default_server; + addr[i].opt.proxy_protocol = proxy_protocol; +#if (NGX_STREAM_SSL) + addr[i].opt.ssl = ssl; +#endif + + return NGX_OK; } -found: + /* add the address to the addresses list that bound to this port */ + + return ngx_stream_add_address(cf, cscf, port, lsopt); +} + + +/* + * add the server address, the server names and the server core module + * configurations to the port list + */ + +static ngx_int_t +ngx_stream_add_address(ngx_conf_t *cf, ngx_stream_core_srv_conf_t *cscf, + ngx_stream_conf_port_t *port, ngx_stream_listen_opt_t *lsopt) +{ + ngx_stream_conf_addr_t *addr; + + if (port->addrs.elts == NULL) { + if (ngx_array_init(&port->addrs, cf->temp_pool, 4, + sizeof(ngx_stream_conf_addr_t)) + != NGX_OK) + { + return NGX_ERROR; + } + } addr = ngx_array_push(&port->addrs); if (addr == NULL) { return NGX_ERROR; } - addr->opt = *listen; + addr->opt = *lsopt; + addr->protocols = 0; + addr->protocols_set = 0; + addr->protocols_changed = 0; + addr->hash.buckets = NULL; + addr->hash.size = 0; + addr->wc_head = NULL; + addr->wc_tail = NULL; +#if (NGX_PCRE) + addr->nregex = 0; + addr->regex = NULL; +#endif + addr->default_server = cscf; + addr->servers.elts = NULL; + + return ngx_stream_add_server(cf, cscf, addr); +} + + +/* add the server core module configuration to the address:port */ + +static ngx_int_t +ngx_stream_add_server(ngx_conf_t *cf, ngx_stream_core_srv_conf_t *cscf, + ngx_stream_conf_addr_t *addr) +{ + ngx_uint_t i; + ngx_stream_core_srv_conf_t **server; + + if (addr->servers.elts == NULL) { + if (ngx_array_init(&addr->servers, cf->temp_pool, 4, + sizeof(ngx_stream_core_srv_conf_t *)) + != NGX_OK) + { + return NGX_ERROR; + } + + } else { + server = addr->servers.elts; + for (i = 0; i < addr->servers.nelts; i++) { + if (server[i] == cscf) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "a duplicate listen %V", + &addr->opt.addr_text); + return NGX_ERROR; + } + } + } + + server = ngx_array_push(&addr->servers); + if (server == NULL) { + return NGX_ERROR; + } + + *server = cscf; return NGX_OK; } -static char * -ngx_stream_optimize_servers(ngx_conf_t *cf, ngx_array_t *ports) +static ngx_int_t +ngx_stream_optimize_servers(ngx_conf_t *cf, ngx_stream_core_main_conf_t *cmcf, + ngx_array_t *ports) { - ngx_uint_t i, p, last, bind_wildcard; - ngx_listening_t *ls; - ngx_stream_port_t *stport; - ngx_stream_conf_port_t *port; - ngx_stream_conf_addr_t *addr; - ngx_stream_core_srv_conf_t *cscf; + ngx_uint_t p, a; + ngx_stream_conf_port_t *port; + ngx_stream_conf_addr_t *addr; + + if (ports == NULL) { + return NGX_OK; + } port = ports->elts; for (p = 0; p < ports->nelts; p++) { @@ -451,103 +681,361 @@ ngx_stream_optimize_servers(ngx_conf_t *cf, ngx_array_t *ports) ngx_sort(port[p].addrs.elts, (size_t) port[p].addrs.nelts, sizeof(ngx_stream_conf_addr_t), ngx_stream_cmp_conf_addrs); - addr = port[p].addrs.elts; - last = port[p].addrs.nelts; - /* - * if there is the binding to the "*:port" then we need to bind() - * to the "*:port" only and ignore the other bindings + * check whether all name-based servers have the same + * configuration as a default server for given address:port */ - if (addr[last - 1].opt.wildcard) { - addr[last - 1].opt.bind = 1; - bind_wildcard = 1; + addr = port[p].addrs.elts; + for (a = 0; a < port[p].addrs.nelts; a++) { - } else { - bind_wildcard = 0; + if (addr[a].servers.nelts > 1 +#if (NGX_PCRE) + || addr[a].default_server->captures +#endif + ) + { + if (ngx_stream_server_names(cf, cmcf, &addr[a]) != NGX_OK) { + return NGX_ERROR; + } + } + } + + if (ngx_stream_init_listening(cf, &port[p]) != NGX_OK) { + return NGX_ERROR; } + } + + return NGX_OK; +} + + +static ngx_int_t +ngx_stream_server_names(ngx_conf_t *cf, ngx_stream_core_main_conf_t *cmcf, + ngx_stream_conf_addr_t *addr) +{ + ngx_int_t rc; + ngx_uint_t n, s; + ngx_hash_init_t hash; + ngx_hash_keys_arrays_t ha; + ngx_stream_server_name_t *name; + ngx_stream_core_srv_conf_t **cscfp; +#if (NGX_PCRE) + ngx_uint_t regex, i; + + regex = 0; +#endif + + ngx_memzero(&ha, sizeof(ngx_hash_keys_arrays_t)); + + ha.temp_pool = ngx_create_pool(NGX_DEFAULT_POOL_SIZE, cf->log); + if (ha.temp_pool == NULL) { + return NGX_ERROR; + } + + ha.pool = cf->pool; + + if (ngx_hash_keys_array_init(&ha, NGX_HASH_LARGE) != NGX_OK) { + goto failed; + } + + cscfp = addr->servers.elts; + + for (s = 0; s < addr->servers.nelts; s++) { - i = 0; + name = cscfp[s]->server_names.elts; - while (i < last) { + for (n = 0; n < cscfp[s]->server_names.nelts; n++) { - if (bind_wildcard && !addr[i].opt.bind) { - i++; +#if (NGX_PCRE) + if (name[n].regex) { + regex++; continue; } +#endif - ls = ngx_create_listening(cf, addr[i].opt.sockaddr, - addr[i].opt.socklen); - if (ls == NULL) { - return NGX_CONF_ERROR; + rc = ngx_hash_add_key(&ha, &name[n].name, name[n].server, + NGX_HASH_WILDCARD_KEY); + + if (rc == NGX_ERROR) { + goto failed; } - ls->addr_ntop = 1; - ls->handler = ngx_stream_init_connection; - ls->pool_size = 256; - ls->type = addr[i].opt.type; + if (rc == NGX_DECLINED) { + ngx_log_error(NGX_LOG_EMERG, cf->log, 0, + "invalid server name or wildcard \"%V\" on %V", + &name[n].name, &addr->opt.addr_text); + goto failed; + } - cscf = addr->opt.ctx->srv_conf[ngx_stream_core_module.ctx_index]; + if (rc == NGX_BUSY) { + ngx_log_error(NGX_LOG_WARN, cf->log, 0, + "conflicting server name \"%V\" on %V, ignored", + &name[n].name, &addr->opt.addr_text); + } + } + } - ls->logp = cscf->error_log; - ls->log.data = &ls->addr_text; - ls->log.handler = ngx_accept_log_error; + hash.key = ngx_hash_key_lc; + hash.max_size = cmcf->server_names_hash_max_size; + hash.bucket_size = cmcf->server_names_hash_bucket_size; + hash.name = "server_names_hash"; + hash.pool = cf->pool; - ls->backlog = addr[i].opt.backlog; - ls->rcvbuf = addr[i].opt.rcvbuf; - ls->sndbuf = addr[i].opt.sndbuf; + if (ha.keys.nelts) { + hash.hash = &addr->hash; + hash.temp_pool = NULL; - ls->wildcard = addr[i].opt.wildcard; + if (ngx_hash_init(&hash, ha.keys.elts, ha.keys.nelts) != NGX_OK) { + goto failed; + } + } - ls->keepalive = addr[i].opt.so_keepalive; -#if (NGX_HAVE_KEEPALIVE_TUNABLE) - ls->keepidle = addr[i].opt.tcp_keepidle; - ls->keepintvl = addr[i].opt.tcp_keepintvl; - ls->keepcnt = addr[i].opt.tcp_keepcnt; -#endif + if (ha.dns_wc_head.nelts) { -#if (NGX_HAVE_INET6) - ls->ipv6only = addr[i].opt.ipv6only; -#endif + ngx_qsort(ha.dns_wc_head.elts, (size_t) ha.dns_wc_head.nelts, + sizeof(ngx_hash_key_t), ngx_stream_cmp_dns_wildcards); -#if (NGX_HAVE_TCP_FASTOPEN) - ls->fastopen = addr[i].opt.fastopen; -#endif + hash.hash = NULL; + hash.temp_pool = ha.temp_pool; -#if (NGX_HAVE_REUSEPORT) - ls->reuseport = addr[i].opt.reuseport; -#endif + if (ngx_hash_wildcard_init(&hash, ha.dns_wc_head.elts, + ha.dns_wc_head.nelts) + != NGX_OK) + { + goto failed; + } - stport = ngx_palloc(cf->pool, sizeof(ngx_stream_port_t)); - if (stport == NULL) { - return NGX_CONF_ERROR; + addr->wc_head = (ngx_hash_wildcard_t *) hash.hash; + } + + if (ha.dns_wc_tail.nelts) { + + ngx_qsort(ha.dns_wc_tail.elts, (size_t) ha.dns_wc_tail.nelts, + sizeof(ngx_hash_key_t), ngx_stream_cmp_dns_wildcards); + + hash.hash = NULL; + hash.temp_pool = ha.temp_pool; + + if (ngx_hash_wildcard_init(&hash, ha.dns_wc_tail.elts, + ha.dns_wc_tail.nelts) + != NGX_OK) + { + goto failed; + } + + addr->wc_tail = (ngx_hash_wildcard_t *) hash.hash; + } + + ngx_destroy_pool(ha.temp_pool); + +#if (NGX_PCRE) + + if (regex == 0) { + return NGX_OK; + } + + addr->nregex = regex; + addr->regex = ngx_palloc(cf->pool, + regex * sizeof(ngx_stream_server_name_t)); + if (addr->regex == NULL) { + return NGX_ERROR; + } + + i = 0; + + for (s = 0; s < addr->servers.nelts; s++) { + + name = cscfp[s]->server_names.elts; + + for (n = 0; n < cscfp[s]->server_names.nelts; n++) { + if (name[n].regex) { + addr->regex[i++] = name[n]; } + } + } + +#endif + + return NGX_OK; + +failed: + + ngx_destroy_pool(ha.temp_pool); + + return NGX_ERROR; +} + + +static ngx_int_t +ngx_stream_cmp_conf_addrs(const void *one, const void *two) +{ + ngx_stream_conf_addr_t *first, *second; + + first = (ngx_stream_conf_addr_t *) one; + second = (ngx_stream_conf_addr_t *) two; + + if (first->opt.wildcard) { + /* a wildcard address must be the last resort, shift it to the end */ + return 1; + } + + if (second->opt.wildcard) { + /* a wildcard address must be the last resort, shift it to the end */ + return -1; + } + + if (first->opt.bind && !second->opt.bind) { + /* shift explicit bind()ed addresses to the start */ + return -1; + } + + if (!first->opt.bind && second->opt.bind) { + /* shift explicit bind()ed addresses to the start */ + return 1; + } + + /* do not sort by default */ + + return 0; +} + + +static int ngx_libc_cdecl +ngx_stream_cmp_dns_wildcards(const void *one, const void *two) +{ + ngx_hash_key_t *first, *second; + + first = (ngx_hash_key_t *) one; + second = (ngx_hash_key_t *) two; + + return ngx_dns_strcmp(first->key.data, second->key.data); +} + + +static ngx_int_t +ngx_stream_init_listening(ngx_conf_t *cf, ngx_stream_conf_port_t *port) +{ + ngx_uint_t i, last, bind_wildcard; + ngx_listening_t *ls; + ngx_stream_port_t *stport; + ngx_stream_conf_addr_t *addr; + + addr = port->addrs.elts; + last = port->addrs.nelts; + + /* + * If there is a binding to an "*:port" then we need to bind() to + * the "*:port" only and ignore other implicit bindings. The bindings + * have been already sorted: explicit bindings are on the start, then + * implicit bindings go, and wildcard binding is in the end. + */ + + if (addr[last - 1].opt.wildcard) { + addr[last - 1].opt.bind = 1; + bind_wildcard = 1; + + } else { + bind_wildcard = 0; + } + + i = 0; + + while (i < last) { + + if (bind_wildcard && !addr[i].opt.bind) { + i++; + continue; + } + + ls = ngx_stream_add_listening(cf, &addr[i]); + if (ls == NULL) { + return NGX_ERROR; + } + + stport = ngx_pcalloc(cf->pool, sizeof(ngx_stream_port_t)); + if (stport == NULL) { + return NGX_ERROR; + } + + ls->servers = stport; - ls->servers = stport; + stport->naddrs = i + 1; - stport->naddrs = i + 1; + switch (ls->sockaddr->sa_family) { - switch (ls->sockaddr->sa_family) { #if (NGX_HAVE_INET6) - case AF_INET6: - if (ngx_stream_add_addrs6(cf, stport, addr) != NGX_OK) { - return NGX_CONF_ERROR; - } - break; + case AF_INET6: + if (ngx_stream_add_addrs6(cf, stport, addr) != NGX_OK) { + return NGX_ERROR; + } + break; #endif - default: /* AF_INET */ - if (ngx_stream_add_addrs(cf, stport, addr) != NGX_OK) { - return NGX_CONF_ERROR; - } - break; + default: /* AF_INET */ + if (ngx_stream_add_addrs(cf, stport, addr) != NGX_OK) { + return NGX_ERROR; } - - addr++; - last--; + break; } + + addr++; + last--; } - return NGX_CONF_OK; + return NGX_OK; +} + + +static ngx_listening_t * +ngx_stream_add_listening(ngx_conf_t *cf, ngx_stream_conf_addr_t *addr) +{ + ngx_listening_t *ls; + ngx_stream_core_srv_conf_t *cscf; + + ls = ngx_create_listening(cf, addr->opt.sockaddr, addr->opt.socklen); + if (ls == NULL) { + return NULL; + } + + ls->addr_ntop = 1; + + ls->handler = ngx_stream_init_connection; + + ls->pool_size = 256; + + cscf = addr->default_server; + + ls->logp = cscf->error_log; + ls->log.data = &ls->addr_text; + ls->log.handler = ngx_accept_log_error; + + ls->type = addr->opt.type; + ls->backlog = addr->opt.backlog; + ls->rcvbuf = addr->opt.rcvbuf; + ls->sndbuf = addr->opt.sndbuf; + + ls->keepalive = addr->opt.so_keepalive; +#if (NGX_HAVE_KEEPALIVE_TUNABLE) + ls->keepidle = addr->opt.tcp_keepidle; + ls->keepintvl = addr->opt.tcp_keepintvl; + ls->keepcnt = addr->opt.tcp_keepcnt; +#endif + +#if (NGX_HAVE_INET6) + ls->ipv6only = addr->opt.ipv6only; +#endif + +#if (NGX_HAVE_TCP_FASTOPEN) + ls->fastopen = addr->opt.fastopen; +#endif + +#if (NGX_HAVE_REUSEPORT) + ls->reuseport = addr->opt.reuseport; +#endif + + ls->wildcard = addr->opt.wildcard; + + return ls; } @@ -555,9 +1043,10 @@ static ngx_int_t ngx_stream_add_addrs(ngx_conf_t *cf, ngx_stream_port_t *stport, ngx_stream_conf_addr_t *addr) { - ngx_uint_t i; - struct sockaddr_in *sin; - ngx_stream_in_addr_t *addrs; + ngx_uint_t i; + struct sockaddr_in *sin; + ngx_stream_in_addr_t *addrs; + ngx_stream_virtual_names_t *vn; stport->addrs = ngx_pcalloc(cf->pool, stport->naddrs * sizeof(ngx_stream_in_addr_t)); @@ -571,13 +1060,39 @@ ngx_stream_add_addrs(ngx_conf_t *cf, ngx_stream_port_t *stport, sin = (struct sockaddr_in *) addr[i].opt.sockaddr; addrs[i].addr = sin->sin_addr.s_addr; - - addrs[i].conf.ctx = addr[i].opt.ctx; + addrs[i].conf.default_server = addr[i].default_server; #if (NGX_STREAM_SSL) addrs[i].conf.ssl = addr[i].opt.ssl; #endif addrs[i].conf.proxy_protocol = addr[i].opt.proxy_protocol; - addrs[i].conf.addr_text = addr[i].opt.addr_text; + + if (addr[i].hash.buckets == NULL + && (addr[i].wc_head == NULL + || addr[i].wc_head->hash.buckets == NULL) + && (addr[i].wc_tail == NULL + || addr[i].wc_tail->hash.buckets == NULL) +#if (NGX_PCRE) + && addr[i].nregex == 0 +#endif + ) + { + continue; + } + + vn = ngx_palloc(cf->pool, sizeof(ngx_stream_virtual_names_t)); + if (vn == NULL) { + return NGX_ERROR; + } + + addrs[i].conf.virtual_names = vn; + + vn->names.hash = addr[i].hash; + vn->names.wc_head = addr[i].wc_head; + vn->names.wc_tail = addr[i].wc_tail; +#if (NGX_PCRE) + vn->nregex = addr[i].nregex; + vn->regex = addr[i].regex; +#endif } return NGX_OK; @@ -590,9 +1105,10 @@ static ngx_int_t ngx_stream_add_addrs6(ngx_conf_t *cf, ngx_stream_port_t *stport, ngx_stream_conf_addr_t *addr) { - ngx_uint_t i; - struct sockaddr_in6 *sin6; - ngx_stream_in6_addr_t *addrs6; + ngx_uint_t i; + struct sockaddr_in6 *sin6; + ngx_stream_in6_addr_t *addrs6; + ngx_stream_virtual_names_t *vn; stport->addrs = ngx_pcalloc(cf->pool, stport->naddrs * sizeof(ngx_stream_in6_addr_t)); @@ -606,50 +1122,42 @@ ngx_stream_add_addrs6(ngx_conf_t *cf, ngx_stream_port_t *stport, sin6 = (struct sockaddr_in6 *) addr[i].opt.sockaddr; addrs6[i].addr6 = sin6->sin6_addr; - - addrs6[i].conf.ctx = addr[i].opt.ctx; + addrs6[i].conf.default_server = addr[i].default_server; #if (NGX_STREAM_SSL) addrs6[i].conf.ssl = addr[i].opt.ssl; #endif addrs6[i].conf.proxy_protocol = addr[i].opt.proxy_protocol; - addrs6[i].conf.addr_text = addr[i].opt.addr_text; - } - - return NGX_OK; -} + if (addr[i].hash.buckets == NULL + && (addr[i].wc_head == NULL + || addr[i].wc_head->hash.buckets == NULL) + && (addr[i].wc_tail == NULL + || addr[i].wc_tail->hash.buckets == NULL) +#if (NGX_PCRE) + && addr[i].nregex == 0 #endif + ) + { + continue; + } + vn = ngx_palloc(cf->pool, sizeof(ngx_stream_virtual_names_t)); + if (vn == NULL) { + return NGX_ERROR; + } -static ngx_int_t -ngx_stream_cmp_conf_addrs(const void *one, const void *two) -{ - ngx_stream_conf_addr_t *first, *second; - - first = (ngx_stream_conf_addr_t *) one; - second = (ngx_stream_conf_addr_t *) two; - - if (first->opt.wildcard) { - /* a wildcard must be the last resort, shift it to the end */ - return 1; - } - - if (second->opt.wildcard) { - /* a wildcard must be the last resort, shift it to the end */ - return -1; - } - - if (first->opt.bind && !second->opt.bind) { - /* shift explicit bind()ed addresses to the start */ - return -1; - } + addrs6[i].conf.virtual_names = vn; - if (!first->opt.bind && second->opt.bind) { - /* shift explicit bind()ed addresses to the start */ - return 1; + vn->names.hash = addr[i].hash; + vn->names.wc_head = addr[i].wc_head; + vn->names.wc_tail = addr[i].wc_tail; +#if (NGX_PCRE) + vn->nregex = addr[i].nregex; + vn->regex = addr[i].regex; +#endif } - /* do not sort by default */ - - return 0; + return NGX_OK; } + +#endif diff --git a/src/stream/ngx_stream.h b/src/stream/ngx_stream.h index 46c362296a6..781e937515c 100644 --- a/src/stream/ngx_stream.h +++ b/src/stream/ngx_stream.h @@ -45,9 +45,8 @@ typedef struct { socklen_t socklen; ngx_str_t addr_text; - /* server ctx */ - ngx_stream_conf_ctx_t *ctx; - + unsigned set:1; + unsigned default_server:1; unsigned bind:1; unsigned wildcard:1; unsigned ssl:1; @@ -69,50 +68,7 @@ typedef struct { int fastopen; #endif int type; -} ngx_stream_listen_t; - - -typedef struct { - ngx_stream_conf_ctx_t *ctx; - ngx_str_t addr_text; - unsigned ssl:1; - unsigned proxy_protocol:1; -} ngx_stream_addr_conf_t; - -typedef struct { - in_addr_t addr; - ngx_stream_addr_conf_t conf; -} ngx_stream_in_addr_t; - - -#if (NGX_HAVE_INET6) - -typedef struct { - struct in6_addr addr6; - ngx_stream_addr_conf_t conf; -} ngx_stream_in6_addr_t; - -#endif - - -typedef struct { - /* ngx_stream_in_addr_t or ngx_stream_in6_addr_t */ - void *addrs; - ngx_uint_t naddrs; -} ngx_stream_port_t; - - -typedef struct { - int family; - int type; - in_port_t port; - ngx_array_t addrs; /* array of ngx_stream_conf_addr_t */ -} ngx_stream_conf_port_t; - - -typedef struct { - ngx_stream_listen_t opt; -} ngx_stream_conf_addr_t; +} ngx_stream_listen_opt_t; typedef enum { @@ -153,7 +109,6 @@ typedef struct { typedef struct { ngx_array_t servers; /* ngx_stream_core_srv_conf_t */ - ngx_array_t listen; /* ngx_stream_listen_t */ ngx_stream_phase_engine_t phase_engine; @@ -163,16 +118,24 @@ typedef struct { ngx_array_t prefix_variables; /* ngx_stream_variable_t */ ngx_uint_t ncaptures; + ngx_uint_t server_names_hash_max_size; + ngx_uint_t server_names_hash_bucket_size; + ngx_uint_t variables_hash_max_size; ngx_uint_t variables_hash_bucket_size; ngx_hash_keys_arrays_t *variables_keys; + ngx_array_t *ports; + ngx_stream_phase_t phases[NGX_STREAM_LOG_PHASE + 1]; } ngx_stream_core_main_conf_t; typedef struct { + /* array of the ngx_stream_server_name_t, "server_name" directive */ + ngx_array_t server_names; + ngx_stream_content_handler_pt handler; ngx_stream_conf_ctx_t *ctx; @@ -180,6 +143,8 @@ typedef struct { u_char *file_name; ngx_uint_t line; + ngx_str_t server_name; + ngx_flag_t tcp_nodelay; size_t preread_buffer_size; ngx_msec_t preread_timeout; @@ -191,10 +156,98 @@ typedef struct { ngx_msec_t proxy_protocol_timeout; - ngx_uint_t listen; /* unsigned listen:1; */ + unsigned listen:1; +#if (NGX_PCRE) + unsigned captures:1; +#endif } ngx_stream_core_srv_conf_t; +/* list of structures to find core_srv_conf quickly at run time */ + + +typedef struct { +#if (NGX_PCRE) + ngx_stream_regex_t *regex; +#endif + ngx_stream_core_srv_conf_t *server; /* virtual name server conf */ + ngx_str_t name; +} ngx_stream_server_name_t; + + +typedef struct { + ngx_hash_combined_t names; + + ngx_uint_t nregex; + ngx_stream_server_name_t *regex; +} ngx_stream_virtual_names_t; + + +typedef struct { + /* the default server configuration for this address:port */ + ngx_stream_core_srv_conf_t *default_server; + + ngx_stream_virtual_names_t *virtual_names; + + unsigned ssl:1; + unsigned proxy_protocol:1; +} ngx_stream_addr_conf_t; + + +typedef struct { + in_addr_t addr; + ngx_stream_addr_conf_t conf; +} ngx_stream_in_addr_t; + + +#if (NGX_HAVE_INET6) + +typedef struct { + struct in6_addr addr6; + ngx_stream_addr_conf_t conf; +} ngx_stream_in6_addr_t; + +#endif + + +typedef struct { + /* ngx_stream_in_addr_t or ngx_stream_in6_addr_t */ + void *addrs; + ngx_uint_t naddrs; +} ngx_stream_port_t; + + +typedef struct { + int family; + int type; + in_port_t port; + ngx_array_t addrs; /* array of ngx_stream_conf_addr_t */ +} ngx_stream_conf_port_t; + + +typedef struct { + ngx_stream_listen_opt_t opt; + + unsigned protocols:3; + unsigned protocols_set:1; + unsigned protocols_changed:1; + + ngx_hash_t hash; + ngx_hash_wildcard_t *wc_head; + ngx_hash_wildcard_t *wc_tail; + +#if (NGX_PCRE) + ngx_uint_t nregex; + ngx_stream_server_name_t *regex; +#endif + + /* the default server configuration for this address:port */ + ngx_stream_core_srv_conf_t *default_server; + ngx_array_t servers; + /* array of ngx_stream_core_srv_conf_t */ +} ngx_stream_conf_addr_t; + + struct ngx_stream_session_s { uint32_t signature; /* "STRM" */ @@ -210,6 +263,8 @@ struct ngx_stream_session_s { void **main_conf; void **srv_conf; + ngx_stream_virtual_names_t *virtual_names; + ngx_stream_upstream_t *upstream; ngx_array_t *upstream_states; /* of ngx_stream_upstream_state_t */ @@ -283,6 +338,9 @@ typedef struct { #define NGX_STREAM_WRITE_BUFFERED 0x10 +ngx_int_t ngx_stream_add_listen(ngx_conf_t *cf, + ngx_stream_core_srv_conf_t *cscf, ngx_stream_listen_opt_t *lsopt); + void ngx_stream_core_run_phases(ngx_stream_session_t *s); ngx_int_t ngx_stream_core_generic_phase(ngx_stream_session_t *s, ngx_stream_phase_handler_t *ph); @@ -291,6 +349,10 @@ ngx_int_t ngx_stream_core_preread_phase(ngx_stream_session_t *s, ngx_int_t ngx_stream_core_content_phase(ngx_stream_session_t *s, ngx_stream_phase_handler_t *ph); +ngx_int_t ngx_stream_validate_host(ngx_str_t *host, ngx_pool_t *pool, + ngx_uint_t alloc); +ngx_int_t ngx_stream_find_virtual_server(ngx_stream_session_t *s, + ngx_str_t *host, ngx_stream_core_srv_conf_t **cscfp); void ngx_stream_init_connection(ngx_connection_t *c); void ngx_stream_session_handler(ngx_event_t *rev); diff --git a/src/stream/ngx_stream_core_module.c b/src/stream/ngx_stream_core_module.c index 2ef39af2425..31a34aee57c 100644 --- a/src/stream/ngx_stream_core_module.c +++ b/src/stream/ngx_stream_core_module.c @@ -27,6 +27,8 @@ static char *ngx_stream_core_server(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); static char *ngx_stream_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); +static char *ngx_stream_core_server_name(ngx_conf_t *cf, ngx_command_t *cmd, + void *conf); static char *ngx_stream_core_resolver(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); @@ -47,6 +49,20 @@ static ngx_command_t ngx_stream_core_commands[] = { offsetof(ngx_stream_core_main_conf_t, variables_hash_bucket_size), NULL }, + { ngx_string("server_names_hash_max_size"), + NGX_STREAM_MAIN_CONF|NGX_CONF_TAKE1, + ngx_conf_set_num_slot, + NGX_STREAM_MAIN_CONF_OFFSET, + offsetof(ngx_stream_core_main_conf_t, server_names_hash_max_size), + NULL }, + + { ngx_string("server_names_hash_bucket_size"), + NGX_STREAM_MAIN_CONF|NGX_CONF_TAKE1, + ngx_conf_set_num_slot, + NGX_STREAM_MAIN_CONF_OFFSET, + offsetof(ngx_stream_core_main_conf_t, server_names_hash_bucket_size), + NULL }, + { ngx_string("server"), NGX_STREAM_MAIN_CONF|NGX_CONF_BLOCK|NGX_CONF_NOARGS, ngx_stream_core_server, @@ -61,6 +77,13 @@ static ngx_command_t ngx_stream_core_commands[] = { 0, NULL }, + { ngx_string("server_name"), + NGX_STREAM_SRV_CONF|NGX_CONF_1MORE, + ngx_stream_core_server_name, + NGX_STREAM_SRV_CONF_OFFSET, + 0, + NULL }, + { ngx_string("error_log"), NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_1MORE, ngx_stream_core_error_log, @@ -441,6 +464,149 @@ ngx_stream_core_content_phase(ngx_stream_session_t *s, } +ngx_int_t +ngx_stream_validate_host(ngx_str_t *host, ngx_pool_t *pool, ngx_uint_t alloc) +{ + u_char *h, ch; + size_t i, dot_pos, host_len; + + enum { + sw_usual = 0, + sw_literal, + sw_rest + } state; + + dot_pos = host->len; + host_len = host->len; + + h = host->data; + + state = sw_usual; + + for (i = 0; i < host->len; i++) { + ch = h[i]; + + switch (ch) { + + case '.': + if (dot_pos == i - 1) { + return NGX_DECLINED; + } + dot_pos = i; + break; + + case ':': + if (state == sw_usual) { + host_len = i; + state = sw_rest; + } + break; + + case '[': + if (i == 0) { + state = sw_literal; + } + break; + + case ']': + if (state == sw_literal) { + host_len = i + 1; + state = sw_rest; + } + break; + + default: + + if (ngx_path_separator(ch)) { + return NGX_DECLINED; + } + + if (ch <= 0x20 || ch == 0x7f) { + return NGX_DECLINED; + } + + if (ch >= 'A' && ch <= 'Z') { + alloc = 1; + } + + break; + } + } + + if (dot_pos == host_len - 1) { + host_len--; + } + + if (host_len == 0) { + return NGX_DECLINED; + } + + if (alloc) { + host->data = ngx_pnalloc(pool, host_len); + if (host->data == NULL) { + return NGX_ERROR; + } + + ngx_strlow(host->data, h, host_len); + } + + host->len = host_len; + + return NGX_OK; +} + + +ngx_int_t +ngx_stream_find_virtual_server(ngx_stream_session_t *s, + ngx_str_t *host, ngx_stream_core_srv_conf_t **cscfp) +{ + ngx_stream_core_srv_conf_t *cscf; + + if (s->virtual_names == NULL) { + return NGX_DECLINED; + } + + cscf = ngx_hash_find_combined(&s->virtual_names->names, + ngx_hash_key(host->data, host->len), + host->data, host->len); + + if (cscf) { + *cscfp = cscf; + return NGX_OK; + } + +#if (NGX_PCRE) + + if (host->len && s->virtual_names->nregex) { + ngx_int_t n; + ngx_uint_t i; + ngx_stream_server_name_t *sn; + + sn = s->virtual_names->regex; + + for (i = 0; i < s->virtual_names->nregex; i++) { + + n = ngx_stream_regex_exec(s, sn[i].regex, host); + + if (n == NGX_DECLINED) { + continue; + } + + if (n == NGX_OK) { + *cscfp = sn[i].server; + return NGX_OK; + } + + return NGX_ERROR; + } + } + +#endif /* NGX_PCRE */ + + return NGX_DECLINED; +} + + static ngx_int_t ngx_stream_core_preconfiguration(ngx_conf_t *cf) { @@ -465,11 +631,8 @@ ngx_stream_core_create_main_conf(ngx_conf_t *cf) return NULL; } - if (ngx_array_init(&cmcf->listen, cf->pool, 4, sizeof(ngx_stream_listen_t)) - != NGX_OK) - { - return NULL; - } + cmcf->server_names_hash_max_size = NGX_CONF_UNSET_UINT; + cmcf->server_names_hash_bucket_size = NGX_CONF_UNSET_UINT; cmcf->variables_hash_max_size = NGX_CONF_UNSET_UINT; cmcf->variables_hash_bucket_size = NGX_CONF_UNSET_UINT; @@ -483,6 +646,14 @@ ngx_stream_core_init_main_conf(ngx_conf_t *cf, void *conf) { ngx_stream_core_main_conf_t *cmcf = conf; + ngx_conf_init_uint_value(cmcf->server_names_hash_max_size, 512); + ngx_conf_init_uint_value(cmcf->server_names_hash_bucket_size, + ngx_cacheline_size); + + cmcf->server_names_hash_bucket_size = + ngx_align(cmcf->server_names_hash_bucket_size, ngx_cacheline_size); + + ngx_conf_init_uint_value(cmcf->variables_hash_max_size, 1024); ngx_conf_init_uint_value(cmcf->variables_hash_bucket_size, 64); @@ -514,6 +685,13 @@ ngx_stream_core_create_srv_conf(ngx_conf_t *cf) * cscf->error_log = NULL; */ + if (ngx_array_init(&cscf->server_names, cf->temp_pool, 4, + sizeof(ngx_stream_server_name_t)) + != NGX_OK) + { + return NULL; + } + cscf->file_name = cf->conf_file->file.name.data; cscf->line = cf->conf_file->line; cscf->resolver_timeout = NGX_CONF_UNSET_MSEC; @@ -532,6 +710,9 @@ ngx_stream_core_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) ngx_stream_core_srv_conf_t *prev = parent; ngx_stream_core_srv_conf_t *conf = child; + ngx_str_t name; + ngx_stream_server_name_t *sn; + ngx_conf_merge_msec_value(conf->resolver_timeout, prev->resolver_timeout, 30000); @@ -579,6 +760,37 @@ ngx_stream_core_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) ngx_conf_merge_msec_value(conf->preread_timeout, prev->preread_timeout, 30000); + if (conf->server_names.nelts == 0) { + /* the array has 4 empty preallocated elements, so push cannot fail */ + sn = ngx_array_push(&conf->server_names); +#if (NGX_PCRE) + sn->regex = NULL; +#endif + sn->server = conf; + ngx_str_set(&sn->name, ""); + } + + sn = conf->server_names.elts; + name = sn[0].name; + +#if (NGX_PCRE) + if (sn->regex) { + name.len++; + name.data--; + } else +#endif + + if (name.data[0] == '.') { + name.len--; + name.data++; + } + + conf->server_name.len = name.len; + conf->server_name.data = ngx_pstrdup(cf->pool, &name); + if (conf->server_name.data == NULL) { + return NGX_CONF_ERROR; + } + return NGX_CONF_OK; } @@ -678,11 +890,10 @@ ngx_stream_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) { ngx_stream_core_srv_conf_t *cscf = conf; - ngx_str_t *value, size; - ngx_url_t u; - ngx_uint_t i, n, backlog; - ngx_stream_listen_t *ls, *als, *nls; - ngx_stream_core_main_conf_t *cmcf; + ngx_str_t *value, size; + ngx_url_t u; + ngx_uint_t n, i, backlog; + ngx_stream_listen_opt_t lsopt; cscf->listen = 1; @@ -703,51 +914,48 @@ ngx_stream_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) return NGX_CONF_ERROR; } - cmcf = ngx_stream_conf_get_module_main_conf(cf, ngx_stream_core_module); - - ls = ngx_array_push(&cmcf->listen); - if (ls == NULL) { - return NGX_CONF_ERROR; - } - - ngx_memzero(ls, sizeof(ngx_stream_listen_t)); - - ls->backlog = NGX_LISTEN_BACKLOG; - ls->rcvbuf = -1; - ls->sndbuf = -1; - ls->type = SOCK_STREAM; - ls->ctx = cf->ctx; + ngx_memzero(&lsopt, sizeof(ngx_stream_listen_opt_t)); + lsopt.backlog = NGX_LISTEN_BACKLOG; + lsopt.type = SOCK_STREAM; + lsopt.rcvbuf = -1; + lsopt.sndbuf = -1; #if (NGX_HAVE_TCP_FASTOPEN) - ls->fastopen = -1; + lsopt.fastopen = -1; #endif - #if (NGX_HAVE_INET6) - ls->ipv6only = 1; + lsopt.ipv6only = 1; #endif backlog = 0; for (i = 2; i < cf->args->nelts; i++) { + if (ngx_strcmp(value[i].data, "default_server") == 0) { + lsopt.default_server = 1; + continue; + } + #if !(NGX_WIN32) if (ngx_strcmp(value[i].data, "udp") == 0) { - ls->type = SOCK_DGRAM; + lsopt.type = SOCK_DGRAM; continue; } #endif if (ngx_strcmp(value[i].data, "bind") == 0) { - ls->bind = 1; + lsopt.set = 1; + lsopt.bind = 1; continue; } #if (NGX_HAVE_TCP_FASTOPEN) if (ngx_strncmp(value[i].data, "fastopen=", 9) == 0) { - ls->fastopen = ngx_atoi(value[i].data + 9, value[i].len - 9); - ls->bind = 1; + lsopt.fastopen = ngx_atoi(value[i].data + 9, value[i].len - 9); + lsopt.set = 1; + lsopt.bind = 1; - if (ls->fastopen == NGX_ERROR) { + if (lsopt.fastopen == NGX_ERROR) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "invalid fastopen \"%V\"", &value[i]); return NGX_CONF_ERROR; @@ -758,10 +966,11 @@ ngx_stream_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) #endif if (ngx_strncmp(value[i].data, "backlog=", 8) == 0) { - ls->backlog = ngx_atoi(value[i].data + 8, value[i].len - 8); - ls->bind = 1; + lsopt.backlog = ngx_atoi(value[i].data + 8, value[i].len - 8); + lsopt.set = 1; + lsopt.bind = 1; - if (ls->backlog == NGX_ERROR || ls->backlog == 0) { + if (lsopt.backlog == NGX_ERROR || lsopt.backlog == 0) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "invalid backlog \"%V\"", &value[i]); return NGX_CONF_ERROR; @@ -776,10 +985,11 @@ ngx_stream_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) size.len = value[i].len - 7; size.data = value[i].data + 7; - ls->rcvbuf = ngx_parse_size(&size); - ls->bind = 1; + lsopt.rcvbuf = ngx_parse_size(&size); + lsopt.set = 1; + lsopt.bind = 1; - if (ls->rcvbuf == NGX_ERROR) { + if (lsopt.rcvbuf == NGX_ERROR) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "invalid rcvbuf \"%V\"", &value[i]); return NGX_CONF_ERROR; @@ -792,10 +1002,11 @@ ngx_stream_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) size.len = value[i].len - 7; size.data = value[i].data + 7; - ls->sndbuf = ngx_parse_size(&size); - ls->bind = 1; + lsopt.sndbuf = ngx_parse_size(&size); + lsopt.set = 1; + lsopt.bind = 1; - if (ls->sndbuf == NGX_ERROR) { + if (lsopt.sndbuf == NGX_ERROR) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "invalid sndbuf \"%V\"", &value[i]); return NGX_CONF_ERROR; @@ -807,10 +1018,10 @@ ngx_stream_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) if (ngx_strncmp(value[i].data, "ipv6only=o", 10) == 0) { #if (NGX_HAVE_INET6 && defined IPV6_V6ONLY) if (ngx_strcmp(&value[i].data[10], "n") == 0) { - ls->ipv6only = 1; + lsopt.ipv6only = 1; } else if (ngx_strcmp(&value[i].data[10], "ff") == 0) { - ls->ipv6only = 0; + lsopt.ipv6only = 0; } else { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, @@ -819,7 +1030,9 @@ ngx_stream_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) return NGX_CONF_ERROR; } - ls->bind = 1; + lsopt.set = 1; + lsopt.bind = 1; + continue; #else ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, @@ -831,8 +1044,9 @@ ngx_stream_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) if (ngx_strcmp(value[i].data, "reuseport") == 0) { #if (NGX_HAVE_REUSEPORT) - ls->reuseport = 1; - ls->bind = 1; + lsopt.reuseport = 1; + lsopt.set = 1; + lsopt.bind = 1; #else ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "reuseport is not supported " @@ -843,17 +1057,7 @@ ngx_stream_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) if (ngx_strcmp(value[i].data, "ssl") == 0) { #if (NGX_STREAM_SSL) - ngx_stream_ssl_conf_t *sslcf; - - sslcf = ngx_stream_conf_get_module_srv_conf(cf, - ngx_stream_ssl_module); - - sslcf->listen = 1; - sslcf->file = cf->conf_file->file.name.data; - sslcf->line = cf->conf_file->line; - - ls->ssl = 1; - + lsopt.ssl = 1; continue; #else ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, @@ -866,10 +1070,10 @@ ngx_stream_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) if (ngx_strncmp(value[i].data, "so_keepalive=", 13) == 0) { if (ngx_strcmp(&value[i].data[13], "on") == 0) { - ls->so_keepalive = 1; + lsopt.so_keepalive = 1; } else if (ngx_strcmp(&value[i].data[13], "off") == 0) { - ls->so_keepalive = 2; + lsopt.so_keepalive = 2; } else { @@ -888,8 +1092,8 @@ ngx_stream_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) if (p > s.data) { s.len = p - s.data; - ls->tcp_keepidle = ngx_parse_time(&s, 1); - if (ls->tcp_keepidle == (time_t) NGX_ERROR) { + lsopt.tcp_keepidle = ngx_parse_time(&s, 1); + if (lsopt.tcp_keepidle == (time_t) NGX_ERROR) { goto invalid_so_keepalive; } } @@ -904,8 +1108,8 @@ ngx_stream_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) if (p > s.data) { s.len = p - s.data; - ls->tcp_keepintvl = ngx_parse_time(&s, 1); - if (ls->tcp_keepintvl == (time_t) NGX_ERROR) { + lsopt.tcp_keepintvl = ngx_parse_time(&s, 1); + if (lsopt.tcp_keepintvl == (time_t) NGX_ERROR) { goto invalid_so_keepalive; } } @@ -915,19 +1119,19 @@ ngx_stream_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) if (s.data < end) { s.len = end - s.data; - ls->tcp_keepcnt = ngx_atoi(s.data, s.len); - if (ls->tcp_keepcnt == NGX_ERROR) { + lsopt.tcp_keepcnt = ngx_atoi(s.data, s.len); + if (lsopt.tcp_keepcnt == NGX_ERROR) { goto invalid_so_keepalive; } } - if (ls->tcp_keepidle == 0 && ls->tcp_keepintvl == 0 - && ls->tcp_keepcnt == 0) + if (lsopt.tcp_keepidle == 0 && lsopt.tcp_keepintvl == 0 + && lsopt.tcp_keepcnt == 0) { goto invalid_so_keepalive; } - ls->so_keepalive = 1; + lsopt.so_keepalive = 1; #else @@ -939,7 +1143,8 @@ ngx_stream_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) #endif } - ls->bind = 1; + lsopt.set = 1; + lsopt.bind = 1; continue; @@ -954,7 +1159,7 @@ ngx_stream_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) } if (ngx_strcmp(value[i].data, "proxy_protocol") == 0) { - ls->proxy_protocol = 1; + lsopt.proxy_protocol = 1; continue; } @@ -963,27 +1168,27 @@ ngx_stream_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) return NGX_CONF_ERROR; } - if (ls->type == SOCK_DGRAM) { + if (lsopt.type == SOCK_DGRAM) { if (backlog) { return "\"backlog\" parameter is incompatible with \"udp\""; } #if (NGX_STREAM_SSL) - if (ls->ssl) { + if (lsopt.ssl) { return "\"ssl\" parameter is incompatible with \"udp\""; } #endif - if (ls->so_keepalive) { + if (lsopt.so_keepalive) { return "\"so_keepalive\" parameter is incompatible with \"udp\""; } - if (ls->proxy_protocol) { + if (lsopt.proxy_protocol) { return "\"proxy_protocol\" parameter is incompatible with \"udp\""; } #if (NGX_HAVE_TCP_FASTOPEN) - if (ls->fastopen != -1) { + if (lsopt.fastopen != -1) { return "\"fastopen\" parameter is incompatible with \"udp\""; } #endif @@ -1000,45 +1205,118 @@ ngx_stream_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) } } - if (n != 0) { - nls = ngx_array_push(&cmcf->listen); - if (nls == NULL) { - return NGX_CONF_ERROR; - } + lsopt.sockaddr = u.addrs[n].sockaddr; + lsopt.socklen = u.addrs[n].socklen; + lsopt.addr_text = u.addrs[n].name; + lsopt.wildcard = ngx_inet_wildcard(lsopt.sockaddr); + + if (ngx_stream_add_listen(cf, cscf, &lsopt) != NGX_OK) { + return NGX_CONF_ERROR; + } + + next: + continue; + } + + return NGX_CONF_OK; +} + + +static char * +ngx_stream_core_server_name(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) +{ + ngx_stream_core_srv_conf_t *cscf = conf; + + u_char ch; + ngx_str_t *value; + ngx_uint_t i; + ngx_stream_server_name_t *sn; + + value = cf->args->elts; + + for (i = 1; i < cf->args->nelts; i++) { + + ch = value[i].data[0]; + + if ((ch == '*' && (value[i].len < 3 || value[i].data[1] != '.')) + || (ch == '.' && value[i].len < 2)) + { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "server name \"%V\" is invalid", &value[i]); + return NGX_CONF_ERROR; + } + + if (ngx_strchr(value[i].data, '/')) { + ngx_conf_log_error(NGX_LOG_WARN, cf, 0, + "server name \"%V\" has suspicious symbols", + &value[i]); + } + + sn = ngx_array_push(&cscf->server_names); + if (sn == NULL) { + return NGX_CONF_ERROR; + } + +#if (NGX_PCRE) + sn->regex = NULL; +#endif + sn->server = cscf; - *nls = *ls; + if (ngx_strcasecmp(value[i].data, (u_char *) "$hostname") == 0) { + sn->name = cf->cycle->hostname; } else { - nls = ls; + sn->name = value[i]; } - nls->sockaddr = u.addrs[n].sockaddr; - nls->socklen = u.addrs[n].socklen; - nls->addr_text = u.addrs[n].name; - nls->wildcard = ngx_inet_wildcard(nls->sockaddr); + if (value[i].data[0] != '~') { + ngx_strlow(sn->name.data, sn->name.data, sn->name.len); + continue; + } - als = cmcf->listen.elts; +#if (NGX_PCRE) + { + u_char *p; + ngx_regex_compile_t rc; + u_char errstr[NGX_MAX_CONF_ERRSTR]; - for (i = 0; i < cmcf->listen.nelts - 1; i++) { - if (nls->type != als[i].type) { - continue; - } + if (value[i].len == 1) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "empty regex in server name \"%V\"", &value[i]); + return NGX_CONF_ERROR; + } - if (ngx_cmp_sockaddr(als[i].sockaddr, als[i].socklen, - nls->sockaddr, nls->socklen, 1) - != NGX_OK) - { - continue; + value[i].len--; + value[i].data++; + + ngx_memzero(&rc, sizeof(ngx_regex_compile_t)); + + rc.pattern = value[i]; + rc.err.len = NGX_MAX_CONF_ERRSTR; + rc.err.data = errstr; + + for (p = value[i].data; p < value[i].data + value[i].len; p++) { + if (*p >= 'A' && *p <= 'Z') { + rc.options = NGX_REGEX_CASELESS; + break; } + } - ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, - "duplicate \"%V\" address and port pair", - &nls->addr_text); + sn->regex = ngx_stream_regex_compile(cf, &rc); + if (sn->regex == NULL) { return NGX_CONF_ERROR; } - next: - continue; + sn->name = value[i]; + cscf->captures = (rc.captures > 0); + } +#else + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "using regex \"%V\" " + "requires PCRE library", &value[i]); + + return NGX_CONF_ERROR; +#endif } return NGX_CONF_OK; diff --git a/src/stream/ngx_stream_handler.c b/src/stream/ngx_stream_handler.c index 669b6a18d49..a7ffc6e618e 100644 --- a/src/stream/ngx_stream_handler.c +++ b/src/stream/ngx_stream_handler.c @@ -30,6 +30,7 @@ ngx_stream_init_connection(ngx_connection_t *c) struct sockaddr_in *sin; ngx_stream_in_addr_t *addr; ngx_stream_session_t *s; + ngx_stream_conf_ctx_t *ctx; ngx_stream_addr_conf_t *addr_conf; #if (NGX_HAVE_INET6) struct sockaddr_in6 *sin6; @@ -121,9 +122,12 @@ ngx_stream_init_connection(ngx_connection_t *c) return; } + ctx = addr_conf->default_server->ctx; + s->signature = NGX_STREAM_MODULE; - s->main_conf = addr_conf->ctx->main_conf; - s->srv_conf = addr_conf->ctx->srv_conf; + s->main_conf = ctx->main_conf; + s->srv_conf = ctx->srv_conf; + s->virtual_names = addr_conf->virtual_names; #if (NGX_STREAM_SSL) s->ssl = addr_conf->ssl; @@ -144,7 +148,7 @@ ngx_stream_init_connection(ngx_connection_t *c) ngx_log_error(NGX_LOG_INFO, c->log, 0, "*%uA %sclient %*s connected to %V", c->number, c->type == SOCK_DGRAM ? "udp " : "", - len, text, &addr_conf->addr_text); + len, text, &c->listening->addr_text); c->log->connection = c->number; c->log->handler = ngx_stream_log_error; diff --git a/src/stream/ngx_stream_ssl_module.c b/src/stream/ngx_stream_ssl_module.c index 1ba1825ce7a..ee5e2f631f7 100644 --- a/src/stream/ngx_stream_ssl_module.c +++ b/src/stream/ngx_stream_ssl_module.c @@ -219,6 +219,13 @@ static ngx_command_t ngx_stream_ssl_commands[] = { offsetof(ngx_stream_ssl_conf_t, conf_commands), &ngx_stream_ssl_conf_command_post }, + { ngx_string("ssl_reject_handshake"), + NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_FLAG, + ngx_conf_set_flag_slot, + NGX_STREAM_SRV_CONF_OFFSET, + offsetof(ngx_stream_ssl_conf_t, reject_handshake), + NULL }, + { ngx_string("ssl_alpn"), NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_1MORE, ngx_stream_ssl_alpn, @@ -458,7 +465,112 @@ ngx_stream_ssl_handshake_handler(ngx_connection_t *c) static int ngx_stream_ssl_servername(ngx_ssl_conn_t *ssl_conn, int *ad, void *arg) { + ngx_int_t rc; + ngx_str_t host; + const char *servername; + ngx_connection_t *c; + ngx_stream_session_t *s; + ngx_stream_ssl_conf_t *sscf; + ngx_stream_core_srv_conf_t *cscf; + + c = ngx_ssl_get_connection(ssl_conn); + + if (c->ssl->handshaked) { + *ad = SSL_AD_NO_RENEGOTIATION; + return SSL_TLSEXT_ERR_ALERT_FATAL; + } + + s = c->data; + + servername = SSL_get_servername(ssl_conn, TLSEXT_NAMETYPE_host_name); + + if (servername == NULL) { + ngx_log_debug0(NGX_LOG_DEBUG_STREAM, c->log, 0, + "SSL server name: null"); + goto done; + } + + ngx_log_debug1(NGX_LOG_DEBUG_STREAM, c->log, 0, + "SSL server name: \"%s\"", servername); + + host.len = ngx_strlen(servername); + + if (host.len == 0) { + goto done; + } + + host.data = (u_char *) servername; + + rc = ngx_stream_validate_host(&host, c->pool, 1); + + if (rc == NGX_ERROR) { + goto error; + } + + if (rc == NGX_DECLINED) { + goto done; + } + + rc = ngx_stream_find_virtual_server(s, &host, &cscf); + + if (rc == NGX_ERROR) { + goto error; + } + + if (rc == NGX_DECLINED) { + goto done; + } + + s->srv_conf = cscf->ctx->srv_conf; + + ngx_set_connection_log(c, cscf->error_log); + + sscf = ngx_stream_get_module_srv_conf(s, ngx_stream_ssl_module); + + if (sscf->ssl.ctx) { + if (SSL_set_SSL_CTX(ssl_conn, sscf->ssl.ctx) == NULL) { + goto error; + } + + /* + * SSL_set_SSL_CTX() only changes certs as of 1.0.0d + * adjust other things we care about + */ + + SSL_set_verify(ssl_conn, SSL_CTX_get_verify_mode(sscf->ssl.ctx), + SSL_CTX_get_verify_callback(sscf->ssl.ctx)); + + SSL_set_verify_depth(ssl_conn, SSL_CTX_get_verify_depth(sscf->ssl.ctx)); + +#if OPENSSL_VERSION_NUMBER >= 0x009080dfL + /* only in 0.9.8m+ */ + SSL_clear_options(ssl_conn, SSL_get_options(ssl_conn) & + ~SSL_CTX_get_options(sscf->ssl.ctx)); +#endif + + SSL_set_options(ssl_conn, SSL_CTX_get_options(sscf->ssl.ctx)); + +#ifdef SSL_OP_NO_RENEGOTIATION + SSL_set_options(ssl_conn, SSL_OP_NO_RENEGOTIATION); +#endif + } + +done: + + sscf = ngx_stream_get_module_srv_conf(s, ngx_stream_ssl_module); + + if (sscf->reject_handshake) { + c->ssl->handshake_rejected = 1; + *ad = SSL_AD_UNRECOGNIZED_NAME; + return SSL_TLSEXT_ERR_ALERT_FATAL; + } + return SSL_TLSEXT_ERR_OK; + +error: + + *ad = SSL_AD_INTERNAL_ERROR; + return SSL_TLSEXT_ERR_ALERT_FATAL; } #endif @@ -655,7 +767,6 @@ ngx_stream_ssl_create_conf(ngx_conf_t *cf) /* * set by ngx_pcalloc(): * - * scf->listen = 0; * scf->protocols = 0; * scf->certificate_values = NULL; * scf->dhparam = { 0, NULL }; @@ -674,6 +785,7 @@ ngx_stream_ssl_create_conf(ngx_conf_t *cf) scf->passwords = NGX_CONF_UNSET_PTR; scf->conf_commands = NGX_CONF_UNSET_PTR; scf->prefer_server_ciphers = NGX_CONF_UNSET; + scf->reject_handshake = NGX_CONF_UNSET; scf->verify = NGX_CONF_UNSET_UINT; scf->verify_depth = NGX_CONF_UNSET_UINT; scf->builtin_session_cache = NGX_CONF_UNSET; @@ -702,6 +814,8 @@ ngx_stream_ssl_merge_conf(ngx_conf_t *cf, void *parent, void *child) ngx_conf_merge_value(conf->prefer_server_ciphers, prev->prefer_server_ciphers, 0); + ngx_conf_merge_value(conf->reject_handshake, prev->reject_handshake, 0); + ngx_conf_merge_bitmask_value(conf->protocols, prev->protocols, (NGX_CONF_BITMASK_SET |NGX_SSL_TLSv1|NGX_SSL_TLSv1_1 @@ -735,35 +849,21 @@ ngx_stream_ssl_merge_conf(ngx_conf_t *cf, void *parent, void *child) conf->ssl.log = cf->log; - if (!conf->listen) { - return NGX_CONF_OK; - } - - if (conf->certificates == NULL) { - ngx_log_error(NGX_LOG_EMERG, cf->log, 0, - "no \"ssl_certificate\" is defined for " - "the \"listen ... ssl\" directive in %s:%ui", - conf->file, conf->line); - return NGX_CONF_ERROR; - } + if (conf->certificates) { - if (conf->certificate_keys == NULL) { - ngx_log_error(NGX_LOG_EMERG, cf->log, 0, - "no \"ssl_certificate_key\" is defined for " - "the \"listen ... ssl\" directive in %s:%ui", - conf->file, conf->line); - return NGX_CONF_ERROR; - } + if (conf->certificate_keys == NULL + || conf->certificate_keys->nelts < conf->certificates->nelts) + { + ngx_log_error(NGX_LOG_EMERG, cf->log, 0, + "no \"ssl_certificate_key\" is defined " + "for certificate \"%V\"", + ((ngx_str_t *) conf->certificates->elts) + + conf->certificates->nelts - 1); + return NGX_CONF_ERROR; + } - if (conf->certificate_keys->nelts < conf->certificates->nelts) { - ngx_log_error(NGX_LOG_EMERG, cf->log, 0, - "no \"ssl_certificate_key\" is defined " - "for certificate \"%V\" and " - "the \"listen ... ssl\" directive in %s:%ui", - ((ngx_str_t *) conf->certificates->elts) - + conf->certificates->nelts - 1, - conf->file, conf->line); - return NGX_CONF_ERROR; + } else if (!conf->reject_handshake) { + return NGX_CONF_OK; } if (ngx_ssl_create(&conf->ssl, conf->protocols, NULL) != NGX_OK) { @@ -818,7 +918,7 @@ ngx_stream_ssl_merge_conf(ngx_conf_t *cf, void *parent, void *child) return NGX_CONF_ERROR; #endif - } else { + } else if (conf->certificates) { /* configure certificates */ @@ -917,6 +1017,10 @@ ngx_stream_ssl_compile_certificates(ngx_conf_t *cf, ngx_stream_complex_value_t *cv; ngx_stream_compile_complex_value_t ccv; + if (conf->certificates == NULL) { + return NGX_OK; + } + cert = conf->certificates->elts; key = conf->certificate_keys->elts; nelts = conf->certificates->nelts; @@ -1195,8 +1299,13 @@ ngx_stream_ssl_conf_command_check(ngx_conf_t *cf, void *post, void *data) static ngx_int_t ngx_stream_ssl_init(ngx_conf_t *cf) { - ngx_stream_handler_pt *h; - ngx_stream_core_main_conf_t *cmcf; + ngx_uint_t a, p, s; + ngx_stream_handler_pt *h; + ngx_stream_ssl_conf_t *sscf; + ngx_stream_conf_addr_t *addr; + ngx_stream_conf_port_t *port; + ngx_stream_core_srv_conf_t **cscfp, *cscf; + ngx_stream_core_main_conf_t *cmcf; cmcf = ngx_stream_conf_get_module_main_conf(cf, ngx_stream_core_module); @@ -1207,5 +1316,58 @@ ngx_stream_ssl_init(ngx_conf_t *cf) *h = ngx_stream_ssl_handler; + if (cmcf->ports == NULL) { + return NGX_OK; + } + + port = cmcf->ports->elts; + for (p = 0; p < cmcf->ports->nelts; p++) { + + addr = port[p].addrs.elts; + for (a = 0; a < port[p].addrs.nelts; a++) { + + if (!addr[a].opt.ssl) { + continue; + } + + cscf = addr[a].default_server; + sscf = cscf->ctx->srv_conf[ngx_stream_ssl_module.ctx_index]; + + if (sscf->certificates) { + continue; + } + + if (!sscf->reject_handshake) { + ngx_log_error(NGX_LOG_EMERG, cf->log, 0, + "no \"ssl_certificate\" is defined for " + "the \"listen ... ssl\" directive in %s:%ui", + cscf->file_name, cscf->line); + return NGX_ERROR; + } + + /* + * if no certificates are defined in the default server, + * check all non-default server blocks + */ + + cscfp = addr[a].servers.elts; + for (s = 0; s < addr[a].servers.nelts; s++) { + + cscf = cscfp[s]; + sscf = cscf->ctx->srv_conf[ngx_stream_ssl_module.ctx_index]; + + if (sscf->certificates || sscf->reject_handshake) { + continue; + } + + ngx_log_error(NGX_LOG_EMERG, cf->log, 0, + "no \"ssl_certificate\" is defined for " + "the \"listen ... ssl\" directive in %s:%ui", + cscf->file_name, cscf->line); + return NGX_ERROR; + } + } + } + return NGX_OK; } diff --git a/src/stream/ngx_stream_ssl_module.h b/src/stream/ngx_stream_ssl_module.h index e7c825e9b8a..3a9cf54af79 100644 --- a/src/stream/ngx_stream_ssl_module.h +++ b/src/stream/ngx_stream_ssl_module.h @@ -18,10 +18,10 @@ typedef struct { ngx_msec_t handshake_timeout; ngx_flag_t prefer_server_ciphers; + ngx_flag_t reject_handshake; ngx_ssl_t ssl; - ngx_uint_t listen; ngx_uint_t protocols; ngx_uint_t verify; @@ -53,9 +53,6 @@ typedef struct { ngx_flag_t session_tickets; ngx_array_t *session_ticket_keys; - - u_char *file; - ngx_uint_t line; } ngx_stream_ssl_conf_t; diff --git a/src/stream/ngx_stream_ssl_preread_module.c b/src/stream/ngx_stream_ssl_preread_module.c index a236fc5551d..bc96adeee7f 100644 --- a/src/stream/ngx_stream_ssl_preread_module.c +++ b/src/stream/ngx_stream_ssl_preread_module.c @@ -33,6 +33,8 @@ typedef struct { static ngx_int_t ngx_stream_ssl_preread_handler(ngx_stream_session_t *s); static ngx_int_t ngx_stream_ssl_preread_parse_record( ngx_stream_ssl_preread_ctx_t *ctx, u_char *pos, u_char *last); +static ngx_int_t ngx_stream_ssl_preread_servername(ngx_stream_session_t *s, + ngx_str_t *servername); static ngx_int_t ngx_stream_ssl_preread_protocol_variable( ngx_stream_session_t *s, ngx_stream_variable_value_t *v, uintptr_t data); static ngx_int_t ngx_stream_ssl_preread_server_name_variable( @@ -187,6 +189,10 @@ ngx_stream_ssl_preread_handler(ngx_stream_session_t *s) return NGX_DECLINED; } + if (rc == NGX_OK) { + return ngx_stream_ssl_preread_servername(s, &ctx->host); + } + if (rc != NGX_AGAIN) { return rc; } @@ -404,9 +410,6 @@ ngx_stream_ssl_preread_parse_record(ngx_stream_ssl_preread_ctx_t *ctx, case sw_sni_host: ctx->host.len = (p[1] << 8) + p[2]; - ngx_log_debug1(NGX_LOG_DEBUG_STREAM, ctx->log, 0, - "ssl preread: SNI hostname \"%V\"", &ctx->host); - state = sw_ext; dst = NULL; size = ext; @@ -496,6 +499,54 @@ ngx_stream_ssl_preread_parse_record(ngx_stream_ssl_preread_ctx_t *ctx, } +static ngx_int_t +ngx_stream_ssl_preread_servername(ngx_stream_session_t *s, + ngx_str_t *servername) +{ + ngx_int_t rc; + ngx_str_t host; + ngx_connection_t *c; + ngx_stream_core_srv_conf_t *cscf; + + c = s->connection; + + ngx_log_debug1(NGX_LOG_DEBUG_STREAM, c->log, 0, + "SSL preread server name: \"%V\"", servername); + + if (servername->len == 0) { + return NGX_OK; + } + + host = *servername; + + rc = ngx_stream_validate_host(&host, c->pool, 1); + + if (rc == NGX_ERROR) { + return NGX_ERROR; + } + + if (rc == NGX_DECLINED) { + return NGX_OK; + } + + rc = ngx_stream_find_virtual_server(s, &host, &cscf); + + if (rc == NGX_ERROR) { + return NGX_ERROR; + } + + if (rc == NGX_DECLINED) { + return NGX_OK; + } + + s->srv_conf = cscf->ctx->srv_conf; + + ngx_set_connection_log(c, cscf->error_log); + + return NGX_OK; +} + + static ngx_int_t ngx_stream_ssl_preread_protocol_variable(ngx_stream_session_t *s, ngx_variable_value_t *v, uintptr_t data) From a168b810e23627070271dc8bab61cfd6f0caddd7 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Wed, 21 Feb 2024 17:36:02 +0400 Subject: [PATCH 101/279] Stream: ngx_stream_pass_module. The module allows to pass connections from Stream to other modules such as HTTP or Mail, as well as back to Stream. Previously, this was only possible with proxying. Connections with preread buffer read out from socket cannot be passed. The module allows selective SSL termination based on SNI. stream { server { listen 8000 default_server; ssl_preread on; ... } server { listen 8000; server_name foo.example.com; pass 127.0.0.1:8001; # to HTTP } server { listen 8000; server_name bar.example.com; ... } } http { server { listen 8001 ssl; ... location / { root html; } } } --- auto/modules | 10 + auto/options | 3 + src/stream/ngx_stream_pass_module.c | 276 ++++++++++++++++++++++++++++ 3 files changed, 289 insertions(+) create mode 100644 src/stream/ngx_stream_pass_module.c diff --git a/auto/modules b/auto/modules index 300d07cc226..1a5e4212d0b 100644 --- a/auto/modules +++ b/auto/modules @@ -1166,6 +1166,16 @@ if [ $STREAM != NO ]; then . auto/module fi + if [ $STREAM_PASS = YES ]; then + ngx_module_name=ngx_stream_pass_module + ngx_module_deps= + ngx_module_srcs=src/stream/ngx_stream_pass_module.c + ngx_module_libs= + ngx_module_link=$STREAM_PASS + + . auto/module + fi + if [ $STREAM_SET = YES ]; then ngx_module_name=ngx_stream_set_module ngx_module_deps= diff --git a/auto/options b/auto/options index 552ef837ef9..6a6e990a0f6 100644 --- a/auto/options +++ b/auto/options @@ -127,6 +127,7 @@ STREAM_GEOIP=NO STREAM_MAP=YES STREAM_SPLIT_CLIENTS=YES STREAM_RETURN=YES +STREAM_PASS=YES STREAM_SET=YES STREAM_UPSTREAM_HASH=YES STREAM_UPSTREAM_LEAST_CONN=YES @@ -337,6 +338,7 @@ use the \"--with-mail_ssl_module\" option instead" --without-stream_split_clients_module) STREAM_SPLIT_CLIENTS=NO ;; --without-stream_return_module) STREAM_RETURN=NO ;; + --without-stream_pass_module) STREAM_PASS=NO ;; --without-stream_set_module) STREAM_SET=NO ;; --without-stream_upstream_hash_module) STREAM_UPSTREAM_HASH=NO ;; @@ -556,6 +558,7 @@ cat << END --without-stream_split_clients_module disable ngx_stream_split_clients_module --without-stream_return_module disable ngx_stream_return_module + --without-stream_pass_module disable ngx_stream_pass_module --without-stream_set_module disable ngx_stream_set_module --without-stream_upstream_hash_module disable ngx_stream_upstream_hash_module diff --git a/src/stream/ngx_stream_pass_module.c b/src/stream/ngx_stream_pass_module.c new file mode 100644 index 00000000000..2ed0a51c2a4 --- /dev/null +++ b/src/stream/ngx_stream_pass_module.c @@ -0,0 +1,276 @@ + +/* + * Copyright (C) Roman Arutyunyan + * Copyright (C) Nginx, Inc. + */ + + +#include +#include +#include + + +typedef struct { + ngx_addr_t *addr; + ngx_stream_complex_value_t *addr_value; +} ngx_stream_pass_srv_conf_t; + + +static void ngx_stream_pass_handler(ngx_stream_session_t *s); +static ngx_int_t ngx_stream_pass_match(ngx_listening_t *ls, ngx_addr_t *addr); +static void *ngx_stream_pass_create_srv_conf(ngx_conf_t *cf); +static char *ngx_stream_pass(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); + + +static ngx_command_t ngx_stream_pass_commands[] = { + + { ngx_string("pass"), + NGX_STREAM_SRV_CONF|NGX_CONF_TAKE1, + ngx_stream_pass, + NGX_STREAM_SRV_CONF_OFFSET, + 0, + NULL }, + + ngx_null_command +}; + + +static ngx_stream_module_t ngx_stream_pass_module_ctx = { + NULL, /* preconfiguration */ + NULL, /* postconfiguration */ + + NULL, /* create main configuration */ + NULL, /* init main configuration */ + + ngx_stream_pass_create_srv_conf, /* create server configuration */ + NULL /* merge server configuration */ +}; + + +ngx_module_t ngx_stream_pass_module = { + NGX_MODULE_V1, + &ngx_stream_pass_module_ctx, /* module context */ + ngx_stream_pass_commands, /* module directives */ + NGX_STREAM_MODULE, /* module type */ + NULL, /* init master */ + NULL, /* init module */ + NULL, /* init process */ + NULL, /* init thread */ + NULL, /* exit thread */ + NULL, /* exit process */ + NULL, /* exit master */ + NGX_MODULE_V1_PADDING +}; + + +static void +ngx_stream_pass_handler(ngx_stream_session_t *s) +{ + ngx_url_t u; + ngx_str_t url; + ngx_addr_t *addr; + ngx_uint_t i; + ngx_listening_t *ls; + ngx_connection_t *c; + ngx_stream_pass_srv_conf_t *pscf; + + c = s->connection; + + c->log->action = "passing connection to port"; + + if (c->buffer && c->buffer->pos != c->buffer->last) { + ngx_log_error(NGX_LOG_ERR, c->log, 0, + "cannot pass connection with preread data"); + goto failed; + } + + pscf = ngx_stream_get_module_srv_conf(s, ngx_stream_pass_module); + + addr = pscf->addr; + + if (addr == NULL) { + if (ngx_stream_complex_value(s, pscf->addr_value, &url) != NGX_OK) { + goto failed; + } + + ngx_memzero(&u, sizeof(ngx_url_t)); + + u.url = url; + u.no_resolve = 1; + + if (ngx_parse_url(c->pool, &u) != NGX_OK) { + if (u.err) { + ngx_log_error(NGX_LOG_ERR, c->log, 0, + "%s in pass \"%V\"", u.err, &u.url); + } + + goto failed; + } + + if (u.naddrs == 0) { + ngx_log_error(NGX_LOG_ERR, c->log, 0, + "no addresses in pass \"%V\"", &u.url); + goto failed; + } + + if (u.no_port) { + ngx_log_error(NGX_LOG_ERR, c->log, 0, + "no port in pass \"%V\"", &u.url); + goto failed; + } + + addr = &u.addrs[0]; + } + + ngx_log_debug1(NGX_LOG_DEBUG_STREAM, c->log, 0, + "stream pass addr: \"%V\"", &addr->name); + + ls = ngx_cycle->listening.elts; + + for (i = 0; i < ngx_cycle->listening.nelts; i++) { + + if (ngx_stream_pass_match(&ls[i], addr) != NGX_OK) { + continue; + } + + c->listening = &ls[i]; + + c->data = NULL; + c->buffer = NULL; + + *c->log = c->listening->log; + c->log->handler = NULL; + c->log->data = NULL; + + c->local_sockaddr = addr->sockaddr; + c->local_socklen = addr->socklen; + + c->listening->handler(c); + + return; + } + + ngx_log_error(NGX_LOG_ERR, c->log, 0, + "port not found for \"%V\"", &addr->name); + + ngx_stream_finalize_session(s, NGX_STREAM_OK); + + return; + +failed: + + ngx_stream_finalize_session(s, NGX_STREAM_INTERNAL_SERVER_ERROR); +} + + +static ngx_int_t +ngx_stream_pass_match(ngx_listening_t *ls, ngx_addr_t *addr) +{ + if (!ls->wildcard) { + return ngx_cmp_sockaddr(ls->sockaddr, ls->socklen, + addr->sockaddr, addr->socklen, 1); + } + + if (ls->sockaddr->sa_family == addr->sockaddr->sa_family + && ngx_inet_get_port(ls->sockaddr) == ngx_inet_get_port(addr->sockaddr)) + { + return NGX_OK; + } + + return NGX_DECLINED; +} + + +static void * +ngx_stream_pass_create_srv_conf(ngx_conf_t *cf) +{ + ngx_stream_pass_srv_conf_t *conf; + + conf = ngx_pcalloc(cf->pool, sizeof(ngx_stream_pass_srv_conf_t)); + if (conf == NULL) { + return NULL; + } + + /* + * set by ngx_pcalloc(): + * + * conf->addr = NULL; + * conf->addr_value = NULL; + */ + + return conf; +} + + +static char * +ngx_stream_pass(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) +{ + ngx_stream_pass_srv_conf_t *pscf = conf; + + ngx_url_t u; + ngx_str_t *value, *url; + ngx_stream_complex_value_t cv; + ngx_stream_core_srv_conf_t *cscf; + ngx_stream_compile_complex_value_t ccv; + + if (pscf->addr || pscf->addr_value) { + return "is duplicate"; + } + + cscf = ngx_stream_conf_get_module_srv_conf(cf, ngx_stream_core_module); + + cscf->handler = ngx_stream_pass_handler; + + value = cf->args->elts; + + url = &value[1]; + + ngx_memzero(&ccv, sizeof(ngx_stream_compile_complex_value_t)); + + ccv.cf = cf; + ccv.value = url; + ccv.complex_value = &cv; + + if (ngx_stream_compile_complex_value(&ccv) != NGX_OK) { + return NGX_CONF_ERROR; + } + + if (cv.lengths) { + pscf->addr_value = ngx_palloc(cf->pool, + sizeof(ngx_stream_complex_value_t)); + if (pscf->addr_value == NULL) { + return NGX_CONF_ERROR; + } + + *pscf->addr_value = cv; + + return NGX_CONF_OK; + } + + ngx_memzero(&u, sizeof(ngx_url_t)); + + u.url = *url; + u.no_resolve = 1; + + if (ngx_parse_url(cf->pool, &u) != NGX_OK) { + if (u.err) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "%s in \"%V\" of the \"pass\" directive", + u.err, &u.url); + } + + return NGX_CONF_ERROR; + } + + if (u.naddrs == 0) { + return "has no addresses"; + } + + if (u.no_port) { + return "has no port"; + } + + pscf->addr = &u.addrs[0]; + + return NGX_CONF_OK; +} From e4a062b18699c18f570c736e7cbb4245b0659bb6 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Fri, 22 Mar 2024 14:18:51 +0400 Subject: [PATCH 102/279] Stream: using ngx_stream_ssl_srv_conf_t *sscf naming convention. Originally, the stream module was developed based on the mail module, following the existing style. Then it was diverged to closely follow the http module development. This change updates style to use sscf naming convention troughout the stream module, which matches the http module code style. No functional changes. --- src/stream/ngx_stream_ssl_module.c | 194 ++++++++++++++--------------- src/stream/ngx_stream_ssl_module.h | 2 +- 2 files changed, 98 insertions(+), 98 deletions(-) diff --git a/src/stream/ngx_stream_ssl_module.c b/src/stream/ngx_stream_ssl_module.c index ee5e2f631f7..ba444776a9d 100644 --- a/src/stream/ngx_stream_ssl_module.c +++ b/src/stream/ngx_stream_ssl_module.c @@ -40,12 +40,12 @@ static ngx_int_t ngx_stream_ssl_variable(ngx_stream_session_t *s, ngx_stream_variable_value_t *v, uintptr_t data); static ngx_int_t ngx_stream_ssl_add_variables(ngx_conf_t *cf); -static void *ngx_stream_ssl_create_conf(ngx_conf_t *cf); -static char *ngx_stream_ssl_merge_conf(ngx_conf_t *cf, void *parent, +static void *ngx_stream_ssl_create_srv_conf(ngx_conf_t *cf); +static char *ngx_stream_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child); static ngx_int_t ngx_stream_ssl_compile_certificates(ngx_conf_t *cf, - ngx_stream_ssl_conf_t *conf); + ngx_stream_ssl_srv_conf_t *conf); static char *ngx_stream_ssl_password_file(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); @@ -90,21 +90,21 @@ static ngx_command_t ngx_stream_ssl_commands[] = { NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_TAKE1, ngx_conf_set_msec_slot, NGX_STREAM_SRV_CONF_OFFSET, - offsetof(ngx_stream_ssl_conf_t, handshake_timeout), + offsetof(ngx_stream_ssl_srv_conf_t, handshake_timeout), NULL }, { ngx_string("ssl_certificate"), NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_TAKE1, ngx_conf_set_str_array_slot, NGX_STREAM_SRV_CONF_OFFSET, - offsetof(ngx_stream_ssl_conf_t, certificates), + offsetof(ngx_stream_ssl_srv_conf_t, certificates), NULL }, { ngx_string("ssl_certificate_key"), NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_TAKE1, ngx_conf_set_str_array_slot, NGX_STREAM_SRV_CONF_OFFSET, - offsetof(ngx_stream_ssl_conf_t, certificate_keys), + offsetof(ngx_stream_ssl_srv_conf_t, certificate_keys), NULL }, { ngx_string("ssl_password_file"), @@ -118,63 +118,63 @@ static ngx_command_t ngx_stream_ssl_commands[] = { NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_TAKE1, ngx_conf_set_str_slot, NGX_STREAM_SRV_CONF_OFFSET, - offsetof(ngx_stream_ssl_conf_t, dhparam), + offsetof(ngx_stream_ssl_srv_conf_t, dhparam), NULL }, { ngx_string("ssl_ecdh_curve"), NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_TAKE1, ngx_conf_set_str_slot, NGX_STREAM_SRV_CONF_OFFSET, - offsetof(ngx_stream_ssl_conf_t, ecdh_curve), + offsetof(ngx_stream_ssl_srv_conf_t, ecdh_curve), NULL }, { ngx_string("ssl_protocols"), NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_1MORE, ngx_conf_set_bitmask_slot, NGX_STREAM_SRV_CONF_OFFSET, - offsetof(ngx_stream_ssl_conf_t, protocols), + offsetof(ngx_stream_ssl_srv_conf_t, protocols), &ngx_stream_ssl_protocols }, { ngx_string("ssl_ciphers"), NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_TAKE1, ngx_conf_set_str_slot, NGX_STREAM_SRV_CONF_OFFSET, - offsetof(ngx_stream_ssl_conf_t, ciphers), + offsetof(ngx_stream_ssl_srv_conf_t, ciphers), NULL }, { ngx_string("ssl_verify_client"), NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_TAKE1, ngx_conf_set_enum_slot, NGX_STREAM_SRV_CONF_OFFSET, - offsetof(ngx_stream_ssl_conf_t, verify), + offsetof(ngx_stream_ssl_srv_conf_t, verify), &ngx_stream_ssl_verify }, { ngx_string("ssl_verify_depth"), NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_TAKE1, ngx_conf_set_num_slot, NGX_STREAM_SRV_CONF_OFFSET, - offsetof(ngx_stream_ssl_conf_t, verify_depth), + offsetof(ngx_stream_ssl_srv_conf_t, verify_depth), NULL }, { ngx_string("ssl_client_certificate"), NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_TAKE1, ngx_conf_set_str_slot, NGX_STREAM_SRV_CONF_OFFSET, - offsetof(ngx_stream_ssl_conf_t, client_certificate), + offsetof(ngx_stream_ssl_srv_conf_t, client_certificate), NULL }, { ngx_string("ssl_trusted_certificate"), NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_TAKE1, ngx_conf_set_str_slot, NGX_STREAM_SRV_CONF_OFFSET, - offsetof(ngx_stream_ssl_conf_t, trusted_certificate), + offsetof(ngx_stream_ssl_srv_conf_t, trusted_certificate), NULL }, { ngx_string("ssl_prefer_server_ciphers"), NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_FLAG, ngx_conf_set_flag_slot, NGX_STREAM_SRV_CONF_OFFSET, - offsetof(ngx_stream_ssl_conf_t, prefer_server_ciphers), + offsetof(ngx_stream_ssl_srv_conf_t, prefer_server_ciphers), NULL }, { ngx_string("ssl_session_cache"), @@ -188,42 +188,42 @@ static ngx_command_t ngx_stream_ssl_commands[] = { NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_FLAG, ngx_conf_set_flag_slot, NGX_STREAM_SRV_CONF_OFFSET, - offsetof(ngx_stream_ssl_conf_t, session_tickets), + offsetof(ngx_stream_ssl_srv_conf_t, session_tickets), NULL }, { ngx_string("ssl_session_ticket_key"), NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_TAKE1, ngx_conf_set_str_array_slot, NGX_STREAM_SRV_CONF_OFFSET, - offsetof(ngx_stream_ssl_conf_t, session_ticket_keys), + offsetof(ngx_stream_ssl_srv_conf_t, session_ticket_keys), NULL }, { ngx_string("ssl_session_timeout"), NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_TAKE1, ngx_conf_set_sec_slot, NGX_STREAM_SRV_CONF_OFFSET, - offsetof(ngx_stream_ssl_conf_t, session_timeout), + offsetof(ngx_stream_ssl_srv_conf_t, session_timeout), NULL }, { ngx_string("ssl_crl"), NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_TAKE1, ngx_conf_set_str_slot, NGX_STREAM_SRV_CONF_OFFSET, - offsetof(ngx_stream_ssl_conf_t, crl), + offsetof(ngx_stream_ssl_srv_conf_t, crl), NULL }, { ngx_string("ssl_conf_command"), NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_TAKE2, ngx_conf_set_keyval_slot, NGX_STREAM_SRV_CONF_OFFSET, - offsetof(ngx_stream_ssl_conf_t, conf_commands), + offsetof(ngx_stream_ssl_srv_conf_t, conf_commands), &ngx_stream_ssl_conf_command_post }, { ngx_string("ssl_reject_handshake"), NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_FLAG, ngx_conf_set_flag_slot, NGX_STREAM_SRV_CONF_OFFSET, - offsetof(ngx_stream_ssl_conf_t, reject_handshake), + offsetof(ngx_stream_ssl_srv_conf_t, reject_handshake), NULL }, { ngx_string("ssl_alpn"), @@ -244,8 +244,8 @@ static ngx_stream_module_t ngx_stream_ssl_module_ctx = { NULL, /* create main configuration */ NULL, /* init main configuration */ - ngx_stream_ssl_create_conf, /* create server configuration */ - ngx_stream_ssl_merge_conf /* merge server configuration */ + ngx_stream_ssl_create_srv_conf, /* create server configuration */ + ngx_stream_ssl_merge_srv_conf /* merge server configuration */ }; @@ -339,11 +339,11 @@ static ngx_str_t ngx_stream_ssl_sess_id_ctx = ngx_string("STREAM"); static ngx_int_t ngx_stream_ssl_handler(ngx_stream_session_t *s) { - long rc; - X509 *cert; - ngx_int_t rv; - ngx_connection_t *c; - ngx_stream_ssl_conf_t *sslcf; + long rc; + X509 *cert; + ngx_int_t rv; + ngx_connection_t *c; + ngx_stream_ssl_srv_conf_t *sscf; if (!s->ssl) { return NGX_OK; @@ -351,23 +351,23 @@ ngx_stream_ssl_handler(ngx_stream_session_t *s) c = s->connection; - sslcf = ngx_stream_get_module_srv_conf(s, ngx_stream_ssl_module); + sscf = ngx_stream_get_module_srv_conf(s, ngx_stream_ssl_module); if (c->ssl == NULL) { c->log->action = "SSL handshaking"; - rv = ngx_stream_ssl_init_connection(&sslcf->ssl, c); + rv = ngx_stream_ssl_init_connection(&sscf->ssl, c); if (rv != NGX_OK) { return rv; } } - if (sslcf->verify) { + if (sscf->verify) { rc = SSL_get_verify_result(c->ssl->connection); if (rc != X509_V_OK - && (sslcf->verify != 3 || !ngx_ssl_verify_error_optional(rc))) + && (sscf->verify != 3 || !ngx_ssl_verify_error_optional(rc))) { ngx_log_error(NGX_LOG_INFO, c->log, 0, "client SSL certificate verify error: (%l:%s)", @@ -378,7 +378,7 @@ ngx_stream_ssl_handler(ngx_stream_session_t *s) return NGX_ERROR; } - if (sslcf->verify == 1) { + if (sscf->verify == 1) { cert = SSL_get_peer_certificate(c->ssl->connection); if (cert == NULL) { @@ -403,7 +403,7 @@ ngx_stream_ssl_init_connection(ngx_ssl_t *ssl, ngx_connection_t *c) { ngx_int_t rc; ngx_stream_session_t *s; - ngx_stream_ssl_conf_t *sslcf; + ngx_stream_ssl_srv_conf_t *sscf; ngx_stream_core_srv_conf_t *cscf; s = c->data; @@ -425,9 +425,9 @@ ngx_stream_ssl_init_connection(ngx_ssl_t *ssl, ngx_connection_t *c) } if (rc == NGX_AGAIN) { - sslcf = ngx_stream_get_module_srv_conf(s, ngx_stream_ssl_module); + sscf = ngx_stream_get_module_srv_conf(s, ngx_stream_ssl_module); - ngx_add_timer(c->read, sslcf->handshake_timeout); + ngx_add_timer(c->read, sscf->handshake_timeout); c->ssl->handler = ngx_stream_ssl_handshake_handler; @@ -470,7 +470,7 @@ ngx_stream_ssl_servername(ngx_ssl_conn_t *ssl_conn, int *ad, void *arg) const char *servername; ngx_connection_t *c; ngx_stream_session_t *s; - ngx_stream_ssl_conf_t *sscf; + ngx_stream_ssl_srv_conf_t *sscf; ngx_stream_core_srv_conf_t *cscf; c = ngx_ssl_get_connection(ssl_conn); @@ -625,7 +625,7 @@ ngx_stream_ssl_certificate(ngx_ssl_conn_t *ssl_conn, void *arg) ngx_uint_t i, nelts; ngx_connection_t *c; ngx_stream_session_t *s; - ngx_stream_ssl_conf_t *sslcf; + ngx_stream_ssl_srv_conf_t *sscf; ngx_stream_complex_value_t *certs, *keys; c = ngx_ssl_get_connection(ssl_conn); @@ -636,11 +636,11 @@ ngx_stream_ssl_certificate(ngx_ssl_conn_t *ssl_conn, void *arg) s = c->data; - sslcf = arg; + sscf = arg; - nelts = sslcf->certificate_values->nelts; - certs = sslcf->certificate_values->elts; - keys = sslcf->certificate_key_values->elts; + nelts = sscf->certificate_values->nelts; + certs = sscf->certificate_values->elts; + keys = sscf->certificate_key_values->elts; for (i = 0; i < nelts; i++) { @@ -659,7 +659,7 @@ ngx_stream_ssl_certificate(ngx_ssl_conn_t *ssl_conn, void *arg) "ssl key: \"%s\"", key.data); if (ngx_ssl_connection_certificate(c, c->pool, &cert, &key, - sslcf->passwords) + sscf->passwords) != NGX_OK) { return 0; @@ -755,53 +755,53 @@ ngx_stream_ssl_add_variables(ngx_conf_t *cf) static void * -ngx_stream_ssl_create_conf(ngx_conf_t *cf) +ngx_stream_ssl_create_srv_conf(ngx_conf_t *cf) { - ngx_stream_ssl_conf_t *scf; + ngx_stream_ssl_srv_conf_t *sscf; - scf = ngx_pcalloc(cf->pool, sizeof(ngx_stream_ssl_conf_t)); - if (scf == NULL) { + sscf = ngx_pcalloc(cf->pool, sizeof(ngx_stream_ssl_srv_conf_t)); + if (sscf == NULL) { return NULL; } /* * set by ngx_pcalloc(): * - * scf->protocols = 0; - * scf->certificate_values = NULL; - * scf->dhparam = { 0, NULL }; - * scf->ecdh_curve = { 0, NULL }; - * scf->client_certificate = { 0, NULL }; - * scf->trusted_certificate = { 0, NULL }; - * scf->crl = { 0, NULL }; - * scf->alpn = { 0, NULL }; - * scf->ciphers = { 0, NULL }; - * scf->shm_zone = NULL; + * sscf->protocols = 0; + * sscf->certificate_values = NULL; + * sscf->dhparam = { 0, NULL }; + * sscf->ecdh_curve = { 0, NULL }; + * sscf->client_certificate = { 0, NULL }; + * sscf->trusted_certificate = { 0, NULL }; + * sscf->crl = { 0, NULL }; + * sscf->alpn = { 0, NULL }; + * sscf->ciphers = { 0, NULL }; + * sscf->shm_zone = NULL; */ - scf->handshake_timeout = NGX_CONF_UNSET_MSEC; - scf->certificates = NGX_CONF_UNSET_PTR; - scf->certificate_keys = NGX_CONF_UNSET_PTR; - scf->passwords = NGX_CONF_UNSET_PTR; - scf->conf_commands = NGX_CONF_UNSET_PTR; - scf->prefer_server_ciphers = NGX_CONF_UNSET; - scf->reject_handshake = NGX_CONF_UNSET; - scf->verify = NGX_CONF_UNSET_UINT; - scf->verify_depth = NGX_CONF_UNSET_UINT; - scf->builtin_session_cache = NGX_CONF_UNSET; - scf->session_timeout = NGX_CONF_UNSET; - scf->session_tickets = NGX_CONF_UNSET; - scf->session_ticket_keys = NGX_CONF_UNSET_PTR; - - return scf; + sscf->handshake_timeout = NGX_CONF_UNSET_MSEC; + sscf->certificates = NGX_CONF_UNSET_PTR; + sscf->certificate_keys = NGX_CONF_UNSET_PTR; + sscf->passwords = NGX_CONF_UNSET_PTR; + sscf->conf_commands = NGX_CONF_UNSET_PTR; + sscf->prefer_server_ciphers = NGX_CONF_UNSET; + sscf->reject_handshake = NGX_CONF_UNSET; + sscf->verify = NGX_CONF_UNSET_UINT; + sscf->verify_depth = NGX_CONF_UNSET_UINT; + sscf->builtin_session_cache = NGX_CONF_UNSET; + sscf->session_timeout = NGX_CONF_UNSET; + sscf->session_tickets = NGX_CONF_UNSET; + sscf->session_ticket_keys = NGX_CONF_UNSET_PTR; + + return sscf; } static char * -ngx_stream_ssl_merge_conf(ngx_conf_t *cf, void *parent, void *child) +ngx_stream_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) { - ngx_stream_ssl_conf_t *prev = parent; - ngx_stream_ssl_conf_t *conf = child; + ngx_stream_ssl_srv_conf_t *prev = parent; + ngx_stream_ssl_srv_conf_t *conf = child; ngx_pool_cleanup_t *cln; @@ -1010,7 +1010,7 @@ ngx_stream_ssl_merge_conf(ngx_conf_t *cf, void *parent, void *child) static ngx_int_t ngx_stream_ssl_compile_certificates(ngx_conf_t *cf, - ngx_stream_ssl_conf_t *conf) + ngx_stream_ssl_srv_conf_t *conf) { ngx_str_t *cert, *key; ngx_uint_t i, nelts; @@ -1099,19 +1099,19 @@ ngx_stream_ssl_compile_certificates(ngx_conf_t *cf, static char * ngx_stream_ssl_password_file(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) { - ngx_stream_ssl_conf_t *scf = conf; + ngx_stream_ssl_srv_conf_t *sscf = conf; ngx_str_t *value; - if (scf->passwords != NGX_CONF_UNSET_PTR) { + if (sscf->passwords != NGX_CONF_UNSET_PTR) { return "is duplicate"; } value = cf->args->elts; - scf->passwords = ngx_ssl_read_password_file(cf, &value[1]); + sscf->passwords = ngx_ssl_read_password_file(cf, &value[1]); - if (scf->passwords == NULL) { + if (sscf->passwords == NULL) { return NGX_CONF_ERROR; } @@ -1122,7 +1122,7 @@ ngx_stream_ssl_password_file(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) static char * ngx_stream_ssl_session_cache(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) { - ngx_stream_ssl_conf_t *scf = conf; + ngx_stream_ssl_srv_conf_t *sscf = conf; size_t len; ngx_str_t *value, name, size; @@ -1134,17 +1134,17 @@ ngx_stream_ssl_session_cache(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) for (i = 1; i < cf->args->nelts; i++) { if (ngx_strcmp(value[i].data, "off") == 0) { - scf->builtin_session_cache = NGX_SSL_NO_SCACHE; + sscf->builtin_session_cache = NGX_SSL_NO_SCACHE; continue; } if (ngx_strcmp(value[i].data, "none") == 0) { - scf->builtin_session_cache = NGX_SSL_NONE_SCACHE; + sscf->builtin_session_cache = NGX_SSL_NONE_SCACHE; continue; } if (ngx_strcmp(value[i].data, "builtin") == 0) { - scf->builtin_session_cache = NGX_SSL_DFLT_BUILTIN_SCACHE; + sscf->builtin_session_cache = NGX_SSL_DFLT_BUILTIN_SCACHE; continue; } @@ -1159,7 +1159,7 @@ ngx_stream_ssl_session_cache(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) goto invalid; } - scf->builtin_session_cache = n; + sscf->builtin_session_cache = n; continue; } @@ -1202,13 +1202,13 @@ ngx_stream_ssl_session_cache(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) return NGX_CONF_ERROR; } - scf->shm_zone = ngx_shared_memory_add(cf, &name, n, + sscf->shm_zone = ngx_shared_memory_add(cf, &name, n, &ngx_stream_ssl_module); - if (scf->shm_zone == NULL) { + if (sscf->shm_zone == NULL) { return NGX_CONF_ERROR; } - scf->shm_zone->init = ngx_ssl_session_cache_init; + sscf->shm_zone->init = ngx_ssl_session_cache_init; continue; } @@ -1216,8 +1216,8 @@ ngx_stream_ssl_session_cache(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) goto invalid; } - if (scf->shm_zone && scf->builtin_session_cache == NGX_CONF_UNSET) { - scf->builtin_session_cache = NGX_SSL_NO_BUILTIN_SCACHE; + if (sscf->shm_zone && sscf->builtin_session_cache == NGX_CONF_UNSET) { + sscf->builtin_session_cache = NGX_SSL_NO_BUILTIN_SCACHE; } return NGX_CONF_OK; @@ -1236,14 +1236,14 @@ ngx_stream_ssl_alpn(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) { #ifdef TLSEXT_TYPE_application_layer_protocol_negotiation - ngx_stream_ssl_conf_t *scf = conf; + ngx_stream_ssl_srv_conf_t *sscf = conf; u_char *p; size_t len; ngx_str_t *value; ngx_uint_t i; - if (scf->alpn.len) { + if (sscf->alpn.len) { return "is duplicate"; } @@ -1260,19 +1260,19 @@ ngx_stream_ssl_alpn(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) len += value[i].len + 1; } - scf->alpn.data = ngx_pnalloc(cf->pool, len); - if (scf->alpn.data == NULL) { + sscf->alpn.data = ngx_pnalloc(cf->pool, len); + if (sscf->alpn.data == NULL) { return NGX_CONF_ERROR; } - p = scf->alpn.data; + p = sscf->alpn.data; for (i = 1; i < cf->args->nelts; i++) { *p++ = value[i].len; p = ngx_cpymem(p, value[i].data, value[i].len); } - scf->alpn.len = len; + sscf->alpn.len = len; return NGX_CONF_OK; @@ -1301,9 +1301,9 @@ ngx_stream_ssl_init(ngx_conf_t *cf) { ngx_uint_t a, p, s; ngx_stream_handler_pt *h; - ngx_stream_ssl_conf_t *sscf; ngx_stream_conf_addr_t *addr; ngx_stream_conf_port_t *port; + ngx_stream_ssl_srv_conf_t *sscf; ngx_stream_core_srv_conf_t **cscfp, *cscf; ngx_stream_core_main_conf_t *cmcf; diff --git a/src/stream/ngx_stream_ssl_module.h b/src/stream/ngx_stream_ssl_module.h index 3a9cf54af79..6f6d9aec3e9 100644 --- a/src/stream/ngx_stream_ssl_module.h +++ b/src/stream/ngx_stream_ssl_module.h @@ -53,7 +53,7 @@ typedef struct { ngx_flag_t session_tickets; ngx_array_t *session_ticket_keys; -} ngx_stream_ssl_conf_t; +} ngx_stream_ssl_srv_conf_t; extern ngx_module_t ngx_stream_ssl_module; From ae1948aa40c48a97f2e171acb84eb04bfcbe1307 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Fri, 22 Mar 2024 14:51:14 +0400 Subject: [PATCH 103/279] Overhauled some diagnostic messages akin to 1b05b9bbcebf. --- src/http/modules/ngx_http_referer_module.c | 2 +- src/http/modules/ngx_http_ssi_filter_module.c | 2 +- src/mail/ngx_mail_core_module.c | 4 ++-- src/stream/ngx_stream_core_module.c | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/http/modules/ngx_http_referer_module.c b/src/http/modules/ngx_http_referer_module.c index 599dd3a1297..11a681b84d4 100644 --- a/src/http/modules/ngx_http_referer_module.c +++ b/src/http/modules/ngx_http_referer_module.c @@ -631,7 +631,7 @@ ngx_http_add_regex_referer(ngx_conf_t *cf, ngx_http_referer_conf_t *rlcf, #else ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, - "the using of the regex \"%V\" requires PCRE library", + "using regex \"%V\" requires PCRE library", name); return NGX_ERROR; diff --git a/src/http/modules/ngx_http_ssi_filter_module.c b/src/http/modules/ngx_http_ssi_filter_module.c index e7601b83e72..0b84bd32249 100644 --- a/src/http/modules/ngx_http_ssi_filter_module.c +++ b/src/http/modules/ngx_http_ssi_filter_module.c @@ -2001,7 +2001,7 @@ ngx_http_ssi_regex_match(ngx_http_request_t *r, ngx_str_t *pattern, #else ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0, - "the using of the regex \"%V\" in SSI requires PCRE library", + "using regex \"%V\" in SSI requires PCRE library", pattern); return NGX_HTTP_SSI_ERROR; diff --git a/src/mail/ngx_mail_core_module.c b/src/mail/ngx_mail_core_module.c index 487c5de8da8..228a8d0a8be 100644 --- a/src/mail/ngx_mail_core_module.c +++ b/src/mail/ngx_mail_core_module.c @@ -441,7 +441,7 @@ ngx_mail_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) continue; #else ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, - "bind ipv6only is not supported " + "ipv6only is not supported " "on this platform"); return NGX_CONF_ERROR; #endif @@ -564,7 +564,7 @@ ngx_mail_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) } ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, - "the invalid \"%V\" parameter", &value[i]); + "invalid parameter \"%V\"", &value[i]); return NGX_CONF_ERROR; } diff --git a/src/stream/ngx_stream_core_module.c b/src/stream/ngx_stream_core_module.c index 31a34aee57c..67129b5ed58 100644 --- a/src/stream/ngx_stream_core_module.c +++ b/src/stream/ngx_stream_core_module.c @@ -1036,7 +1036,7 @@ ngx_stream_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) continue; #else ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, - "bind ipv6only is not supported " + "ipv6only is not supported " "on this platform"); return NGX_CONF_ERROR; #endif @@ -1164,7 +1164,7 @@ ngx_stream_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) } ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, - "the invalid \"%V\" parameter", &value[i]); + "invalid parameter \"%V\"", &value[i]); return NGX_CONF_ERROR; } From f00b43167abbf8b34c5e1f9edcf8a71cdce8a70e Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Fri, 22 Mar 2024 14:53:19 +0400 Subject: [PATCH 104/279] Stream: reshuffled ngx_stream_listen_opt_t fields. In preparation for adding more parameters to the listen directive, and to be in sync with the corresponding structure in the http module. No functional changes. --- src/stream/ngx_stream.h | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/stream/ngx_stream.h b/src/stream/ngx_stream.h index 781e937515c..3033206da22 100644 --- a/src/stream/ngx_stream.h +++ b/src/stream/ngx_stream.h @@ -56,18 +56,19 @@ typedef struct { unsigned reuseport:1; unsigned so_keepalive:2; unsigned proxy_protocol:1; -#if (NGX_HAVE_KEEPALIVE_TUNABLE) - int tcp_keepidle; - int tcp_keepintvl; - int tcp_keepcnt; -#endif + int backlog; int rcvbuf; int sndbuf; + int type; #if (NGX_HAVE_TCP_FASTOPEN) int fastopen; #endif - int type; +#if (NGX_HAVE_KEEPALIVE_TUNABLE) + int tcp_keepidle; + int tcp_keepintvl; + int tcp_keepcnt; +#endif } ngx_stream_listen_opt_t; From 03eba69013eb4f4499a5c09f72338ab95ce00801 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Fri, 22 Mar 2024 14:53:19 +0400 Subject: [PATCH 105/279] Stream: the "deferred" parameter of the "listen" directive. The Linux TCP_DEFER_ACCEPT support. --- src/stream/ngx_stream.c | 4 ++++ src/stream/ngx_stream.h | 1 + src/stream/ngx_stream_core_module.c | 19 +++++++++++++++++++ 3 files changed, 24 insertions(+) diff --git a/src/stream/ngx_stream.c b/src/stream/ngx_stream.c index 9e16cd382f1..9a918421311 100644 --- a/src/stream/ngx_stream.c +++ b/src/stream/ngx_stream.c @@ -1021,6 +1021,10 @@ ngx_stream_add_listening(ngx_conf_t *cf, ngx_stream_conf_addr_t *addr) ls->keepcnt = addr->opt.tcp_keepcnt; #endif +#if (NGX_HAVE_DEFERRED_ACCEPT && defined TCP_DEFER_ACCEPT) + ls->deferred_accept = addr->opt.deferred_accept; +#endif + #if (NGX_HAVE_INET6) ls->ipv6only = addr->opt.ipv6only; #endif diff --git a/src/stream/ngx_stream.h b/src/stream/ngx_stream.h index 3033206da22..2221a14c943 100644 --- a/src/stream/ngx_stream.h +++ b/src/stream/ngx_stream.h @@ -53,6 +53,7 @@ typedef struct { #if (NGX_HAVE_INET6) unsigned ipv6only:1; #endif + unsigned deferred_accept:1; unsigned reuseport:1; unsigned so_keepalive:2; unsigned proxy_protocol:1; diff --git a/src/stream/ngx_stream_core_module.c b/src/stream/ngx_stream_core_module.c index 67129b5ed58..4bf0784d53a 100644 --- a/src/stream/ngx_stream_core_module.c +++ b/src/stream/ngx_stream_core_module.c @@ -1015,6 +1015,19 @@ ngx_stream_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) continue; } + if (ngx_strcmp(value[i].data, "deferred") == 0) { +#if (NGX_HAVE_DEFERRED_ACCEPT && defined TCP_DEFER_ACCEPT) + lsopt.deferred_accept = 1; + lsopt.set = 1; + lsopt.bind = 1; +#else + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "the deferred accept is not supported " + "on this platform, ignored"); +#endif + continue; + } + if (ngx_strncmp(value[i].data, "ipv6only=o", 10) == 0) { #if (NGX_HAVE_INET6 && defined IPV6_V6ONLY) if (ngx_strcmp(&value[i].data[10], "n") == 0) { @@ -1173,6 +1186,12 @@ ngx_stream_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) return "\"backlog\" parameter is incompatible with \"udp\""; } +#if (NGX_HAVE_DEFERRED_ACCEPT && defined TCP_DEFER_ACCEPT) + if (lsopt.deferred_accept) { + return "\"deferred\" parameter is incompatible with \"udp\""; + } +#endif + #if (NGX_STREAM_SSL) if (lsopt.ssl) { return "\"ssl\" parameter is incompatible with \"udp\""; From 04b9bfe55d414bfed8a7aefa3162d81af26532b4 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Fri, 22 Mar 2024 14:53:19 +0400 Subject: [PATCH 106/279] Stream: the "accept_filter" parameter of the "listen" directive. The FreeBSD accept filters support. --- src/stream/ngx_stream.c | 4 ++++ src/stream/ngx_stream.h | 4 ++++ src/stream/ngx_stream_core_module.c | 20 ++++++++++++++++++++ 3 files changed, 28 insertions(+) diff --git a/src/stream/ngx_stream.c b/src/stream/ngx_stream.c index 9a918421311..e0879d09256 100644 --- a/src/stream/ngx_stream.c +++ b/src/stream/ngx_stream.c @@ -1021,6 +1021,10 @@ ngx_stream_add_listening(ngx_conf_t *cf, ngx_stream_conf_addr_t *addr) ls->keepcnt = addr->opt.tcp_keepcnt; #endif +#if (NGX_HAVE_DEFERRED_ACCEPT && defined SO_ACCEPTFILTER) + ls->accept_filter = addr->opt.accept_filter; +#endif + #if (NGX_HAVE_DEFERRED_ACCEPT && defined TCP_DEFER_ACCEPT) ls->deferred_accept = addr->opt.deferred_accept; #endif diff --git a/src/stream/ngx_stream.h b/src/stream/ngx_stream.h index 2221a14c943..888715888e1 100644 --- a/src/stream/ngx_stream.h +++ b/src/stream/ngx_stream.h @@ -70,6 +70,10 @@ typedef struct { int tcp_keepintvl; int tcp_keepcnt; #endif + +#if (NGX_HAVE_DEFERRED_ACCEPT && defined SO_ACCEPTFILTER) + char *accept_filter; +#endif } ngx_stream_listen_opt_t; diff --git a/src/stream/ngx_stream_core_module.c b/src/stream/ngx_stream_core_module.c index 4bf0784d53a..df08208a78f 100644 --- a/src/stream/ngx_stream_core_module.c +++ b/src/stream/ngx_stream_core_module.c @@ -1015,6 +1015,20 @@ ngx_stream_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) continue; } + if (ngx_strncmp(value[i].data, "accept_filter=", 14) == 0) { +#if (NGX_HAVE_DEFERRED_ACCEPT && defined SO_ACCEPTFILTER) + lsopt.accept_filter = (char *) &value[i].data[14]; + lsopt.set = 1; + lsopt.bind = 1; +#else + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "accept filters \"%V\" are not supported " + "on this platform, ignored", + &value[i]); +#endif + continue; + } + if (ngx_strcmp(value[i].data, "deferred") == 0) { #if (NGX_HAVE_DEFERRED_ACCEPT && defined TCP_DEFER_ACCEPT) lsopt.deferred_accept = 1; @@ -1186,6 +1200,12 @@ ngx_stream_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) return "\"backlog\" parameter is incompatible with \"udp\""; } +#if (NGX_HAVE_DEFERRED_ACCEPT && defined SO_ACCEPTFILTER) + if (lsopt.accept_filter) { + return "\"accept_filter\" parameter is incompatible with \"udp\""; + } +#endif + #if (NGX_HAVE_DEFERRED_ACCEPT && defined TCP_DEFER_ACCEPT) if (lsopt.deferred_accept) { return "\"deferred\" parameter is incompatible with \"udp\""; From bd190d825ceeacf105a2392d88734f21df35d89a Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Fri, 22 Mar 2024 14:53:19 +0400 Subject: [PATCH 107/279] Stream: the "setfib" parameter of the "listen" directive. The FreeBSD SO_SETFIB support. --- src/stream/ngx_stream.c | 4 ++++ src/stream/ngx_stream.h | 3 +++ src/stream/ngx_stream_core_module.c | 19 +++++++++++++++++++ 3 files changed, 26 insertions(+) diff --git a/src/stream/ngx_stream.c b/src/stream/ngx_stream.c index e0879d09256..b6eeb23af31 100644 --- a/src/stream/ngx_stream.c +++ b/src/stream/ngx_stream.c @@ -1033,6 +1033,10 @@ ngx_stream_add_listening(ngx_conf_t *cf, ngx_stream_conf_addr_t *addr) ls->ipv6only = addr->opt.ipv6only; #endif +#if (NGX_HAVE_SETFIB) + ls->setfib = addr->opt.setfib; +#endif + #if (NGX_HAVE_TCP_FASTOPEN) ls->fastopen = addr->opt.fastopen; #endif diff --git a/src/stream/ngx_stream.h b/src/stream/ngx_stream.h index 888715888e1..dc05dc5bac6 100644 --- a/src/stream/ngx_stream.h +++ b/src/stream/ngx_stream.h @@ -62,6 +62,9 @@ typedef struct { int rcvbuf; int sndbuf; int type; +#if (NGX_HAVE_SETFIB) + int setfib; +#endif #if (NGX_HAVE_TCP_FASTOPEN) int fastopen; #endif diff --git a/src/stream/ngx_stream_core_module.c b/src/stream/ngx_stream_core_module.c index df08208a78f..576a2fb160e 100644 --- a/src/stream/ngx_stream_core_module.c +++ b/src/stream/ngx_stream_core_module.c @@ -920,6 +920,9 @@ ngx_stream_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) lsopt.type = SOCK_STREAM; lsopt.rcvbuf = -1; lsopt.sndbuf = -1; +#if (NGX_HAVE_SETFIB) + lsopt.setfib = -1; +#endif #if (NGX_HAVE_TCP_FASTOPEN) lsopt.fastopen = -1; #endif @@ -949,6 +952,22 @@ ngx_stream_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) continue; } +#if (NGX_HAVE_SETFIB) + if (ngx_strncmp(value[i].data, "setfib=", 7) == 0) { + lsopt.setfib = ngx_atoi(value[i].data + 7, value[i].len - 7); + lsopt.set = 1; + lsopt.bind = 1; + + if (lsopt.setfib == NGX_ERROR) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "invalid setfib \"%V\"", &value[i]); + return NGX_CONF_ERROR; + } + + continue; + } +#endif + #if (NGX_HAVE_TCP_FASTOPEN) if (ngx_strncmp(value[i].data, "fastopen=", 9) == 0) { lsopt.fastopen = ngx_atoi(value[i].data + 9, value[i].len - 9); From 45e166b4a4a0e8d0e0f8adeb3438e4745261e8da Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Thu, 18 Jan 2024 19:12:38 +0400 Subject: [PATCH 108/279] Stream: moved fastopen compatibility check. The move makes the code look similar to the corresponding code in http module. --- src/stream/ngx_stream_core_module.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/stream/ngx_stream_core_module.c b/src/stream/ngx_stream_core_module.c index 576a2fb160e..3093963b7df 100644 --- a/src/stream/ngx_stream_core_module.c +++ b/src/stream/ngx_stream_core_module.c @@ -1215,6 +1215,12 @@ ngx_stream_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) } if (lsopt.type == SOCK_DGRAM) { +#if (NGX_HAVE_TCP_FASTOPEN) + if (lsopt.fastopen != -1) { + return "\"fastopen\" parameter is incompatible with \"udp\""; + } +#endif + if (backlog) { return "\"backlog\" parameter is incompatible with \"udp\""; } @@ -1244,12 +1250,6 @@ ngx_stream_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) if (lsopt.proxy_protocol) { return "\"proxy_protocol\" parameter is incompatible with \"udp\""; } - -#if (NGX_HAVE_TCP_FASTOPEN) - if (lsopt.fastopen != -1) { - return "\"fastopen\" parameter is incompatible with \"udp\""; - } -#endif } for (n = 0; n < u.naddrs; n++) { From 5e79d98a59b6d094145200976077aa7ca34a84d0 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Fri, 22 Mar 2024 18:39:50 +0400 Subject: [PATCH 109/279] Stream: $server_name. --- src/stream/ngx_stream_variables.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/stream/ngx_stream_variables.c b/src/stream/ngx_stream_variables.c index 81268909810..4658104e12f 100644 --- a/src/stream/ngx_stream_variables.c +++ b/src/stream/ngx_stream_variables.c @@ -29,6 +29,8 @@ static ngx_int_t ngx_stream_variable_server_addr(ngx_stream_session_t *s, ngx_stream_variable_value_t *v, uintptr_t data); static ngx_int_t ngx_stream_variable_server_port(ngx_stream_session_t *s, ngx_stream_variable_value_t *v, uintptr_t data); +static ngx_int_t ngx_stream_variable_server_name(ngx_stream_session_t *s, + ngx_stream_variable_value_t *v, uintptr_t data); static ngx_int_t ngx_stream_variable_bytes(ngx_stream_session_t *s, ngx_stream_variable_value_t *v, uintptr_t data); static ngx_int_t ngx_stream_variable_session_time(ngx_stream_session_t *s, @@ -91,6 +93,9 @@ static ngx_stream_variable_t ngx_stream_core_variables[] = { { ngx_string("server_port"), NULL, ngx_stream_variable_server_port, 0, 0, 0 }, + { ngx_string("server_name"), NULL, ngx_stream_variable_server_name, + 0, 0, 0 }, + { ngx_string("bytes_sent"), NULL, ngx_stream_variable_bytes, 0, 0, 0 }, @@ -721,6 +726,24 @@ ngx_stream_variable_server_port(ngx_stream_session_t *s, } +static ngx_int_t +ngx_stream_variable_server_name(ngx_stream_session_t *s, + ngx_stream_variable_value_t *v, uintptr_t data) +{ + ngx_stream_core_srv_conf_t *cscf; + + cscf = ngx_stream_get_module_srv_conf(s, ngx_stream_core_module); + + v->len = cscf->server_name.len; + v->valid = 1; + v->no_cacheable = 0; + v->not_found = 0; + v->data = cscf->server_name.data; + + return NGX_OK; +} + + static ngx_int_t ngx_stream_variable_bytes(ngx_stream_session_t *s, ngx_stream_variable_value_t *v, uintptr_t data) From d3d64cacb3ce96477d354fe17d3b5c6e348f933a Mon Sep 17 00:00:00 2001 From: Piotr Sikora Date: Thu, 14 Mar 2024 18:37:20 +0400 Subject: [PATCH 110/279] Geo: fixed uninitialized memory access. While copying ngx_http_variable_value_t structures to geo binary base in ngx_http_geo_copy_values(), and similarly in the stream module, uninitialized parts of these structures are copied as well. These include the "escape" field and possible holes. Calculating crc32 of this data triggers uninitialized memory access. Found with MemorySanitizer. Signed-off-by: Piotr Sikora --- src/http/modules/ngx_http_geo_module.c | 4 +--- src/stream/ngx_stream_geo_module.c | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/http/modules/ngx_http_geo_module.c b/src/http/modules/ngx_http_geo_module.c index ef4e9b84aca..8496b651af8 100644 --- a/src/http/modules/ngx_http_geo_module.c +++ b/src/http/modules/ngx_http_geo_module.c @@ -1259,7 +1259,7 @@ ngx_http_geo_value(ngx_conf_t *cf, ngx_http_geo_conf_ctx_t *ctx, return gvvn->value; } - val = ngx_palloc(ctx->pool, sizeof(ngx_http_variable_value_t)); + val = ngx_pcalloc(ctx->pool, sizeof(ngx_http_variable_value_t)); if (val == NULL) { return NULL; } @@ -1271,8 +1271,6 @@ ngx_http_geo_value(ngx_conf_t *cf, ngx_http_geo_conf_ctx_t *ctx, } val->valid = 1; - val->no_cacheable = 0; - val->not_found = 0; gvvn = ngx_palloc(ctx->temp_pool, sizeof(ngx_http_geo_variable_value_node_t)); diff --git a/src/stream/ngx_stream_geo_module.c b/src/stream/ngx_stream_geo_module.c index 4b4cad8fce6..a9e10100f7b 100644 --- a/src/stream/ngx_stream_geo_module.c +++ b/src/stream/ngx_stream_geo_module.c @@ -1209,7 +1209,7 @@ ngx_stream_geo_value(ngx_conf_t *cf, ngx_stream_geo_conf_ctx_t *ctx, return gvvn->value; } - val = ngx_palloc(ctx->pool, sizeof(ngx_stream_variable_value_t)); + val = ngx_pcalloc(ctx->pool, sizeof(ngx_stream_variable_value_t)); if (val == NULL) { return NULL; } @@ -1221,8 +1221,6 @@ ngx_stream_geo_value(ngx_conf_t *cf, ngx_stream_geo_conf_ctx_t *ctx, } val->valid = 1; - val->no_cacheable = 0; - val->not_found = 0; gvvn = ngx_palloc(ctx->temp_pool, sizeof(ngx_stream_geo_variable_value_node_t)); From 3d5a356abb4f06b0f103290bd31a4c146233956b Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Mon, 18 Mar 2024 17:14:30 +0400 Subject: [PATCH 111/279] Fixed undefined behaviour with IPv4-mapped IPv6 addresses. Previously, it could result when left-shifting signed integer due to implicit integer promotion, such that the most significant bit appeared on the sign bit. In practice, though, this results in the same left value as with an explicit cast, at least on known compilers, such as GCC and Clang. The reason is that in_addr_t, which is equivalent to uint32_t and same as "unsigned int" in ILP32 and LP64 data type models, has the same type width as the intermediate after integer promotion, so there's no side effects such as sign-extension. This explains why adding an explicit cast does not change object files in practice. Found with UndefinedBehaviorSanitizer (shift). Based on a patch by Piotr Sikora. --- src/core/ngx_inet.c | 2 +- src/http/modules/ngx_http_access_module.c | 2 +- src/http/modules/ngx_http_geo_module.c | 4 ++-- src/http/modules/ngx_http_geoip_module.c | 2 +- src/stream/ngx_stream_access_module.c | 2 +- src/stream/ngx_stream_geo_module.c | 4 ++-- src/stream/ngx_stream_geoip_module.c | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/core/ngx_inet.c b/src/core/ngx_inet.c index 4228504adeb..acb2ef48a1e 100644 --- a/src/core/ngx_inet.c +++ b/src/core/ngx_inet.c @@ -507,7 +507,7 @@ ngx_cidr_match(struct sockaddr *sa, ngx_array_t *cidrs) p = inaddr6->s6_addr; - inaddr = p[12] << 24; + inaddr = (in_addr_t) p[12] << 24; inaddr += p[13] << 16; inaddr += p[14] << 8; inaddr += p[15]; diff --git a/src/http/modules/ngx_http_access_module.c b/src/http/modules/ngx_http_access_module.c index 7355de9e735..ea755200d0b 100644 --- a/src/http/modules/ngx_http_access_module.c +++ b/src/http/modules/ngx_http_access_module.c @@ -148,7 +148,7 @@ ngx_http_access_handler(ngx_http_request_t *r) p = sin6->sin6_addr.s6_addr; if (alcf->rules && IN6_IS_ADDR_V4MAPPED(&sin6->sin6_addr)) { - addr = p[12] << 24; + addr = (in_addr_t) p[12] << 24; addr += p[13] << 16; addr += p[14] << 8; addr += p[15]; diff --git a/src/http/modules/ngx_http_geo_module.c b/src/http/modules/ngx_http_geo_module.c index 8496b651af8..75c03978a4c 100644 --- a/src/http/modules/ngx_http_geo_module.c +++ b/src/http/modules/ngx_http_geo_module.c @@ -199,7 +199,7 @@ ngx_http_geo_cidr_variable(ngx_http_request_t *r, ngx_http_variable_value_t *v, p = inaddr6->s6_addr; if (IN6_IS_ADDR_V4MAPPED(inaddr6)) { - inaddr = p[12] << 24; + inaddr = (in_addr_t) p[12] << 24; inaddr += p[13] << 16; inaddr += p[14] << 8; inaddr += p[15]; @@ -272,7 +272,7 @@ ngx_http_geo_range_variable(ngx_http_request_t *r, ngx_http_variable_value_t *v, if (IN6_IS_ADDR_V4MAPPED(inaddr6)) { p = inaddr6->s6_addr; - inaddr = p[12] << 24; + inaddr = (in_addr_t) p[12] << 24; inaddr += p[13] << 16; inaddr += p[14] << 8; inaddr += p[15]; diff --git a/src/http/modules/ngx_http_geoip_module.c b/src/http/modules/ngx_http_geoip_module.c index eaf98764f87..5b8b11fc79c 100644 --- a/src/http/modules/ngx_http_geoip_module.c +++ b/src/http/modules/ngx_http_geoip_module.c @@ -266,7 +266,7 @@ ngx_http_geoip_addr(ngx_http_request_t *r, ngx_http_geoip_conf_t *gcf) if (IN6_IS_ADDR_V4MAPPED(inaddr6)) { p = inaddr6->s6_addr; - inaddr = p[12] << 24; + inaddr = (in_addr_t) p[12] << 24; inaddr += p[13] << 16; inaddr += p[14] << 8; inaddr += p[15]; diff --git a/src/stream/ngx_stream_access_module.c b/src/stream/ngx_stream_access_module.c index a3020d4fbd6..070c22614f5 100644 --- a/src/stream/ngx_stream_access_module.c +++ b/src/stream/ngx_stream_access_module.c @@ -144,7 +144,7 @@ ngx_stream_access_handler(ngx_stream_session_t *s) p = sin6->sin6_addr.s6_addr; if (ascf->rules && IN6_IS_ADDR_V4MAPPED(&sin6->sin6_addr)) { - addr = p[12] << 24; + addr = (in_addr_t) p[12] << 24; addr += p[13] << 16; addr += p[14] << 8; addr += p[15]; diff --git a/src/stream/ngx_stream_geo_module.c b/src/stream/ngx_stream_geo_module.c index a9e10100f7b..2324bef0df6 100644 --- a/src/stream/ngx_stream_geo_module.c +++ b/src/stream/ngx_stream_geo_module.c @@ -190,7 +190,7 @@ ngx_stream_geo_cidr_variable(ngx_stream_session_t *s, p = inaddr6->s6_addr; if (IN6_IS_ADDR_V4MAPPED(inaddr6)) { - inaddr = p[12] << 24; + inaddr = (in_addr_t) p[12] << 24; inaddr += p[13] << 16; inaddr += p[14] << 8; inaddr += p[15]; @@ -263,7 +263,7 @@ ngx_stream_geo_range_variable(ngx_stream_session_t *s, if (IN6_IS_ADDR_V4MAPPED(inaddr6)) { p = inaddr6->s6_addr; - inaddr = p[12] << 24; + inaddr = (in_addr_t) p[12] << 24; inaddr += p[13] << 16; inaddr += p[14] << 8; inaddr += p[15]; diff --git a/src/stream/ngx_stream_geoip_module.c b/src/stream/ngx_stream_geoip_module.c index 6507b716ec1..3ee8f0e33e6 100644 --- a/src/stream/ngx_stream_geoip_module.c +++ b/src/stream/ngx_stream_geoip_module.c @@ -236,7 +236,7 @@ ngx_stream_geoip_addr(ngx_stream_session_t *s, ngx_stream_geoip_conf_t *gcf) if (IN6_IS_ADDR_V4MAPPED(inaddr6)) { p = inaddr6->s6_addr; - inaddr = p[12] << 24; + inaddr = (in_addr_t) p[12] << 24; inaddr += p[13] << 16; inaddr += p[14] << 8; inaddr += p[15]; From 2f9e8431e696b27ffcd69e455a1ee006ca14f8e3 Mon Sep 17 00:00:00 2001 From: Piotr Sikora Date: Mon, 26 Feb 2024 20:00:28 +0000 Subject: [PATCH 112/279] Rewrite: fixed "return" directive without response text. Previously, the response text wasn't initialized and the rewrite module was sending response body set to NULL. Found with UndefinedBehaviorSanitizer (pointer-overflow). Signed-off-by: Piotr Sikora --- src/http/modules/ngx_http_rewrite_module.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/http/modules/ngx_http_rewrite_module.c b/src/http/modules/ngx_http_rewrite_module.c index 0e6d4df64d0..ff3b68716d4 100644 --- a/src/http/modules/ngx_http_rewrite_module.c +++ b/src/http/modules/ngx_http_rewrite_module.c @@ -489,6 +489,7 @@ ngx_http_rewrite_return(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) } if (cf->args->nelts == 2) { + ngx_str_set(&ret->text.value, ""); return NGX_CONF_OK; } From eff2ea1d69a589c5b0a6ddde9ff400ab43377c2f Mon Sep 17 00:00:00 2001 From: Piotr Sikora Date: Mon, 26 Feb 2024 20:00:35 +0000 Subject: [PATCH 113/279] Win32: fixed unique file index calculations. The old code was breaking strict aliasing rules. Signed-off-by: Piotr Sikora --- src/os/win32/ngx_files.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/os/win32/ngx_files.h b/src/os/win32/ngx_files.h index 6e59a6fc97a..78ba13a0e95 100644 --- a/src/os/win32/ngx_files.h +++ b/src/os/win32/ngx_files.h @@ -154,7 +154,8 @@ ngx_int_t ngx_file_info(u_char *filename, ngx_file_info_t *fi); (((off_t) (fi)->nFileSizeHigh << 32) | (fi)->nFileSizeLow) #define ngx_file_fs_size(fi) ngx_file_size(fi) -#define ngx_file_uniq(fi) (*(ngx_file_uniq_t *) &(fi)->nFileIndexHigh) +#define ngx_file_uniq(fi) \ + (((ngx_file_uniq_t) (fi)->nFileIndexHigh << 32) | (fi)->nFileIndexLow) /* 116444736000000000 is commented in src/os/win32/ngx_time.c */ From 818f98da1c46785bc23a99efff653febb1e35ac2 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Mon, 26 Feb 2024 20:00:38 +0000 Subject: [PATCH 114/279] Configure: fixed Linux crypt_r() test to add libcrypt. Previously, the resulting binary was successfully linked because libcrypt was added in a separate test for crypt(). Patch by Piotr Sikora. --- auto/os/linux | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/auto/os/linux b/auto/os/linux index 02dcaf29054..bc0556b3a12 100644 --- a/auto/os/linux +++ b/auto/os/linux @@ -228,6 +228,10 @@ ngx_feature_test="struct crypt_data cd; crypt_r(\"key\", \"salt\", &cd);" . auto/feature +if [ $ngx_found = yes ]; then + CRYPT_LIB="-lcrypt" +fi + ngx_include="sys/vfs.h"; . auto/include From 1bc19fe2db3afc7a72eb0a91e554aee9160e0d00 Mon Sep 17 00:00:00 2001 From: Piotr Sikora Date: Mon, 26 Feb 2024 20:00:40 +0000 Subject: [PATCH 115/279] Detect cache line size at runtime on macOS. Notably, Apple Silicon CPUs have 128 byte cache line size, which is twice the default configured for generic aarch64. Signed-off-by: Piotr Sikora --- src/os/unix/ngx_darwin_init.c | 16 +++++++++++----- src/os/unix/ngx_posix_init.c | 5 ++++- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/os/unix/ngx_darwin_init.c b/src/os/unix/ngx_darwin_init.c index aabe02ff963..70748ee57ab 100644 --- a/src/os/unix/ngx_darwin_init.c +++ b/src/os/unix/ngx_darwin_init.c @@ -9,11 +9,12 @@ #include -char ngx_darwin_kern_ostype[16]; -char ngx_darwin_kern_osrelease[128]; -int ngx_darwin_hw_ncpu; -int ngx_darwin_kern_ipc_somaxconn; -u_long ngx_darwin_net_inet_tcp_sendspace; +char ngx_darwin_kern_ostype[16]; +char ngx_darwin_kern_osrelease[128]; +int ngx_darwin_hw_ncpu; +int ngx_darwin_kern_ipc_somaxconn; +u_long ngx_darwin_net_inet_tcp_sendspace; +int64_t ngx_darwin_hw_cachelinesize; ngx_uint_t ngx_debug_malloc; @@ -56,6 +57,10 @@ sysctl_t sysctls[] = { &ngx_darwin_kern_ipc_somaxconn, sizeof(ngx_darwin_kern_ipc_somaxconn), 0 }, + { "hw.cachelinesize", + &ngx_darwin_hw_cachelinesize, + sizeof(ngx_darwin_hw_cachelinesize), 0 }, + { NULL, NULL, 0, 0 } }; @@ -155,6 +160,7 @@ ngx_os_specific_init(ngx_log_t *log) return NGX_ERROR; } + ngx_cacheline_size = ngx_darwin_hw_cachelinesize; ngx_ncpu = ngx_darwin_hw_ncpu; if (ngx_darwin_kern_ipc_somaxconn > 32767) { diff --git a/src/os/unix/ngx_posix_init.c b/src/os/unix/ngx_posix_init.c index 7824735d0ba..6ac931c0ce2 100644 --- a/src/os/unix/ngx_posix_init.c +++ b/src/os/unix/ngx_posix_init.c @@ -51,7 +51,10 @@ ngx_os_init(ngx_log_t *log) } ngx_pagesize = getpagesize(); - ngx_cacheline_size = NGX_CPU_CACHE_LINE; + + if (ngx_cacheline_size == 0) { + ngx_cacheline_size = NGX_CPU_CACHE_LINE; + } for (n = ngx_pagesize; n >>= 1; ngx_pagesize_shift++) { /* void */ } From 6b1bb998c96278a56d767bc23520c385ab9f3038 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Wed, 27 Mar 2024 19:36:51 +0400 Subject: [PATCH 116/279] Configure: set cache line size for more architectures. Based on a patch by Piotr Sikora. --- auto/os/conf | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/auto/os/conf b/auto/os/conf index d7f6e03824b..bb0ce4ef2b9 100644 --- a/auto/os/conf +++ b/auto/os/conf @@ -115,6 +115,21 @@ case "$NGX_MACHINE" in NGX_MACH_CACHE_LINE=64 ;; + ppc64* | powerpc64*) + have=NGX_ALIGNMENT value=16 . auto/define + NGX_MACH_CACHE_LINE=128 + ;; + + riscv64) + have=NGX_ALIGNMENT value=16 . auto/define + NGX_MACH_CACHE_LINE=64 + ;; + + s390x) + have=NGX_ALIGNMENT value=16 . auto/define + NGX_MACH_CACHE_LINE=256 + ;; + *) have=NGX_ALIGNMENT value=16 . auto/define NGX_MACH_CACHE_LINE=32 From 2deded362ee052564e7359d53c81973c16b18e72 Mon Sep 17 00:00:00 2001 From: Piotr Sikora Date: Mon, 26 Feb 2024 20:00:43 +0000 Subject: [PATCH 117/279] Configure: added support for Homebrew on Apple Silicon. Signed-off-by: Piotr Sikora --- auto/lib/geoip/conf | 17 +++++++++++++++++ auto/lib/google-perftools/conf | 16 ++++++++++++++++ auto/lib/libgd/conf | 17 +++++++++++++++++ auto/lib/openssl/conf | 18 ++++++++++++++++++ auto/lib/pcre/conf | 16 ++++++++++++++++ 5 files changed, 84 insertions(+) diff --git a/auto/lib/geoip/conf b/auto/lib/geoip/conf index 8302aae1714..47165b15bad 100644 --- a/auto/lib/geoip/conf +++ b/auto/lib/geoip/conf @@ -64,6 +64,23 @@ if [ $ngx_found = no ]; then fi +if [ $ngx_found = no ]; then + + # Homebrew on Apple Silicon + + ngx_feature="GeoIP library in /opt/homebrew/" + ngx_feature_path="/opt/homebrew/include" + + if [ $NGX_RPATH = YES ]; then + ngx_feature_libs="-R/opt/homebrew/lib -L/opt/homebrew/lib -lGeoIP" + else + ngx_feature_libs="-L/opt/homebrew/lib -lGeoIP" + fi + + . auto/feature +fi + + if [ $ngx_found = yes ]; then CORE_INCS="$CORE_INCS $ngx_feature_path" diff --git a/auto/lib/google-perftools/conf b/auto/lib/google-perftools/conf index 7f1a9113916..94dadac625d 100644 --- a/auto/lib/google-perftools/conf +++ b/auto/lib/google-perftools/conf @@ -46,6 +46,22 @@ if [ $ngx_found = no ]; then fi +if [ $ngx_found = no ]; then + + # Homebrew on Apple Silicon + + ngx_feature="Google perftools in /opt/homebrew/" + + if [ $NGX_RPATH = YES ]; then + ngx_feature_libs="-R/opt/homebrew/lib -L/opt/homebrew/lib -lprofiler" + else + ngx_feature_libs="-L/opt/homebrew/lib -lprofiler" + fi + + . auto/feature +fi + + if [ $ngx_found = yes ]; then CORE_LIBS="$CORE_LIBS $ngx_feature_libs" diff --git a/auto/lib/libgd/conf b/auto/lib/libgd/conf index 678639767a6..07f565677ee 100644 --- a/auto/lib/libgd/conf +++ b/auto/lib/libgd/conf @@ -65,6 +65,23 @@ if [ $ngx_found = no ]; then fi +if [ $ngx_found = no ]; then + + # Homebrew on Apple Silicon + + ngx_feature="GD library in /opt/homebrew/" + ngx_feature_path="/opt/homebrew/include" + + if [ $NGX_RPATH = YES ]; then + ngx_feature_libs="-R/opt/homebrew/lib -L/opt/homebrew/lib -lgd" + else + ngx_feature_libs="-L/opt/homebrew/lib -lgd" + fi + + . auto/feature +fi + + if [ $ngx_found = yes ]; then CORE_INCS="$CORE_INCS $ngx_feature_path" diff --git a/auto/lib/openssl/conf b/auto/lib/openssl/conf index eda1c0f4ad6..fdf430dff75 100644 --- a/auto/lib/openssl/conf +++ b/auto/lib/openssl/conf @@ -122,6 +122,24 @@ else . auto/feature fi + if [ $ngx_found = no ]; then + + # Homebrew on Apple Silicon + + ngx_feature="OpenSSL library in /opt/homebrew/" + ngx_feature_path="/opt/homebrew/include" + + if [ $NGX_RPATH = YES ]; then + ngx_feature_libs="-R/opt/homebrew/lib -L/opt/homebrew/lib -lssl -lcrypto" + else + ngx_feature_libs="-L/opt/homebrew/lib -lssl -lcrypto" + fi + + ngx_feature_libs="$ngx_feature_libs $NGX_LIBDL $NGX_LIBPTHREAD" + + . auto/feature + fi + if [ $ngx_found = yes ]; then have=NGX_SSL . auto/have CORE_INCS="$CORE_INCS $ngx_feature_path" diff --git a/auto/lib/pcre/conf b/auto/lib/pcre/conf index 20c1cafbe54..cdf1809f594 100644 --- a/auto/lib/pcre/conf +++ b/auto/lib/pcre/conf @@ -182,6 +182,22 @@ else . auto/feature fi + if [ $ngx_found = no ]; then + + # Homebrew on Apple Silicon + + ngx_feature="PCRE library in /opt/homebrew/" + ngx_feature_path="/opt/homebrew/include" + + if [ $NGX_RPATH = YES ]; then + ngx_feature_libs="-R/opt/homebrew/lib -L/opt/homebrew/lib -lpcre" + else + ngx_feature_libs="-L/opt/homebrew/lib -lpcre" + fi + + . auto/feature + fi + if [ $ngx_found = yes ]; then CORE_INCS="$CORE_INCS $ngx_feature_path" CORE_LIBS="$CORE_LIBS $ngx_feature_libs" From b595a68df91bee1eb0fa2841837dcf83ae9174f2 Mon Sep 17 00:00:00 2001 From: Piotr Sikora Date: Mon, 26 Feb 2024 20:00:46 +0000 Subject: [PATCH 118/279] Configure: fixed "make install" when cross-compiling to Windows. Signed-off-by: Piotr Sikora --- auto/install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auto/install b/auto/install index c764fdd2f77..7f73e4bffd3 100644 --- a/auto/install +++ b/auto/install @@ -112,7 +112,7 @@ install: build $NGX_INSTALL_PERL_MODULES test ! -f '\$(DESTDIR)$NGX_SBIN_PATH' \\ || mv '\$(DESTDIR)$NGX_SBIN_PATH' \\ '\$(DESTDIR)$NGX_SBIN_PATH.old' - cp $NGX_OBJS/nginx '\$(DESTDIR)$NGX_SBIN_PATH' + cp $NGX_OBJS/nginx$ngx_binext '\$(DESTDIR)$NGX_SBIN_PATH' test -d '\$(DESTDIR)$NGX_CONF_PREFIX' \\ || mkdir -p '\$(DESTDIR)$NGX_CONF_PREFIX' From e3207a17f084c9eb7905ca0f2cfdb1df088fd165 Mon Sep 17 00:00:00 2001 From: Piotr Sikora Date: Mon, 26 Feb 2024 20:00:48 +0000 Subject: [PATCH 119/279] Configure: allow cross-compiling to Windows using Clang. Signed-off-by: Piotr Sikora --- auto/os/win32 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auto/os/win32 b/auto/os/win32 index b821ae6d811..bce764b54bb 100644 --- a/auto/os/win32 +++ b/auto/os/win32 @@ -18,7 +18,7 @@ ngx_binext=".exe" case "$NGX_CC_NAME" in - gcc) + clang | gcc) CORE_LIBS="$CORE_LIBS -ladvapi32 -lws2_32" MAIN_LINK="$MAIN_LINK -Wl,--export-all-symbols" MAIN_LINK="$MAIN_LINK -Wl,--out-implib=$NGX_OBJS/libnginx.a" From 92f99685717e857de9ffa96993601a90803eb0d8 Mon Sep 17 00:00:00 2001 From: Vladimir Khomutov Date: Wed, 10 Apr 2024 09:38:10 +0300 Subject: [PATCH 120/279] QUIC: fixed close timer processing with early data. The ngx_quic_run() function uses qc->close timer to limit the handshake duration. Normally it is removed by ngx_quic_do_init_streams() which is called once when we are done with initial SSL processing. The problem happens when the client sends early data and streams are initialized in the ngx_quic_run() -> ngx_quic_handle_datagram() call. The order of set/remove timer calls is now reversed; the close timer is set up and the timer fires when assigned, starting the unexpected connection close process. The fix is to skip setting the timer if streams were initialized during handling of the initial datagram. The idle timer for quic is set anyway, and stream-related timeouts are managed by application layer. --- src/event/quic/ngx_event_quic.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/event/quic/ngx_event_quic.c b/src/event/quic/ngx_event_quic.c index 4687c849e42..e4690f7dddc 100644 --- a/src/event/quic/ngx_event_quic.c +++ b/src/event/quic/ngx_event_quic.c @@ -211,7 +211,10 @@ ngx_quic_run(ngx_connection_t *c, ngx_quic_conf_t *conf) qc = ngx_quic_get_connection(c); ngx_add_timer(c->read, qc->tp.max_idle_timeout); - ngx_add_timer(&qc->close, qc->conf->handshake_timeout); + + if (!qc->streams.initialized) { + ngx_add_timer(&qc->close, qc->conf->handshake_timeout); + } ngx_quic_connstate_dbg(c); From bf3e6538b93375e8ac49d3863f74ee7cbe588111 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Thu, 11 Apr 2024 11:37:30 +0400 Subject: [PATCH 121/279] Stream pass: limited the number of passes per connection. Previously a cycle in pass configuration resulted in stack overflow. --- src/stream/ngx_stream_pass_module.c | 51 +++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/src/stream/ngx_stream_pass_module.c b/src/stream/ngx_stream_pass_module.c index 2ed0a51c2a4..2c1c60c6a83 100644 --- a/src/stream/ngx_stream_pass_module.c +++ b/src/stream/ngx_stream_pass_module.c @@ -10,6 +10,9 @@ #include +#define NGX_STREAM_PASS_MAX_PASSES 10 + + typedef struct { ngx_addr_t *addr; ngx_stream_complex_value_t *addr_value; @@ -17,6 +20,8 @@ typedef struct { static void ngx_stream_pass_handler(ngx_stream_session_t *s); +static ngx_int_t ngx_stream_pass_check_cycle(ngx_connection_t *c); +static void ngx_stream_pass_cleanup(void *data); static ngx_int_t ngx_stream_pass_match(ngx_listening_t *ls, ngx_addr_t *addr); static void *ngx_stream_pass_create_srv_conf(ngx_conf_t *cf); static char *ngx_stream_pass(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); @@ -125,6 +130,10 @@ ngx_stream_pass_handler(ngx_stream_session_t *s) ngx_log_debug1(NGX_LOG_DEBUG_STREAM, c->log, 0, "stream pass addr: \"%V\"", &addr->name); + if (ngx_stream_pass_check_cycle(c) != NGX_OK) { + goto failed; + } + ls = ngx_cycle->listening.elts; for (i = 0; i < ngx_cycle->listening.nelts; i++) { @@ -163,6 +172,48 @@ ngx_stream_pass_handler(ngx_stream_session_t *s) } +static ngx_int_t +ngx_stream_pass_check_cycle(ngx_connection_t *c) +{ + ngx_uint_t *num; + ngx_pool_cleanup_t *cln; + + for (cln = c->pool->cleanup; cln; cln = cln->next) { + if (cln->handler != ngx_stream_pass_cleanup) { + continue; + } + + num = cln->data; + + if (++(*num) > NGX_STREAM_PASS_MAX_PASSES) { + ngx_log_error(NGX_LOG_ERR, c->log, 0, "stream pass cycle"); + return NGX_ERROR; + } + + return NGX_OK; + } + + cln = ngx_pool_cleanup_add(c->pool, sizeof(ngx_uint_t)); + if (cln == NULL) { + return NGX_ERROR; + } + + cln->handler = ngx_stream_pass_cleanup; + + num = cln->data; + *num = 1; + + return NGX_OK; +} + + +static void +ngx_stream_pass_cleanup(void *data) +{ + return; +} + + static ngx_int_t ngx_stream_pass_match(ngx_listening_t *ls, ngx_addr_t *addr) { From 14f8190ce7ade4e8471d78a69244330314dff400 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Tue, 16 Apr 2024 18:27:50 +0400 Subject: [PATCH 122/279] nginx-1.25.5-RELEASE --- docs/xml/nginx/changes.xml | 77 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/docs/xml/nginx/changes.xml b/docs/xml/nginx/changes.xml index 6d11d6da406..e71dd083326 100644 --- a/docs/xml/nginx/changes.xml +++ b/docs/xml/nginx/changes.xml @@ -5,6 +5,83 @@ + + + + +виртуальные сервера в модуле stream. + + +virtual servers in the stream module. + + + + + +модуль ngx_stream_pass_module. + + +the ngx_stream_pass_module. + + + + + +параметры deferred, accept_filter и setfib директивы listen в модуле stream. + + +the "deferred", "accept_filter", and "setfib" parameters of the "listen" +directive in the stream module. + + + + + +определение размера строки кеша процессора для некоторых архитектур.
+Спасибо Piotr Sikora. + + +cache line size detection for some architectures.
+Thanks to Piotr Sikora. + + + + + +поддержка Homebrew на Apple Silicon.
+Спасибо Piotr Sikora. + + +support for Homebrew on Apple Silicon.
+Thanks to Piotr Sikora. + + + + + +улучшения и исправления кросс-компиляции для Windows.
+Спасибо Piotr Sikora. + + +Windows cross-compilation bugfixes and improvements.
+Thanks to Piotr Sikora. + + + + + +неожиданное закрытие соединения при использовании 0-RTT в QUIC.
+Спасибо Владимиру Хомутову. + + +unexpected connection closure while using 0-RTT in QUIC.
+Thanks to Vladimir Khomutov. + + + + + + From d8a849ae3c99ee5ca82c9a06074761e937dac6d6 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Tue, 16 Apr 2024 18:29:59 +0400 Subject: [PATCH 123/279] release-1.25.5 tag --- .hgtags | 1 + 1 file changed, 1 insertion(+) diff --git a/.hgtags b/.hgtags index 387b38397f0..4a7ee26780a 100644 --- a/.hgtags +++ b/.hgtags @@ -477,3 +477,4 @@ f8134640e8615448205785cf00b0bc810489b495 release-1.25.1 1d839f05409d1a50d0f15a2bf36547001f99ae40 release-1.25.2 294a3d07234f8f65d7b0e0b0e2c5b05c12c5da0a release-1.25.3 173a0a7dbce569adbb70257c6ec4f0f6bc585009 release-1.25.4 +8618e4d900cc71082fbe7dc72af087937d64faf5 release-1.25.5 From 3f0fe15a98c2f849cf1f3f86a60b547e12273f4c Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Fri, 3 May 2024 20:28:22 +0400 Subject: [PATCH 124/279] Version bump. --- src/core/nginx.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/nginx.h b/src/core/nginx.h index f18c3930fc5..d21b118eea3 100644 --- a/src/core/nginx.h +++ b/src/core/nginx.h @@ -9,8 +9,8 @@ #define _NGINX_H_INCLUDED_ -#define nginx_version 1025005 -#define NGINX_VERSION "1.25.5" +#define nginx_version 1027000 +#define NGINX_VERSION "1.27.0" #define NGINX_VER "nginx/" NGINX_VERSION #ifdef NGX_BUILD From a7e3cd52e0a03286267177aa9b88d64232fbaeaf Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Fri, 3 May 2024 20:28:32 +0400 Subject: [PATCH 125/279] HTTP/3: fixed handling of malformed request body length. Previously, a request body larger than declared in Content-Length resulted in a 413 status code, because Content-Length was mistakenly used as the maximum allowed request body, similar to client_max_body_size. Following the HTTP/3 specification, such requests are now rejected with the 400 error as malformed. --- src/http/v3/ngx_http_v3_request.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/http/v3/ngx_http_v3_request.c b/src/http/v3/ngx_http_v3_request.c index 87f5f3214cd..aec122be612 100644 --- a/src/http/v3/ngx_http_v3_request.c +++ b/src/http/v3/ngx_http_v3_request.c @@ -1575,6 +1575,15 @@ ngx_http_v3_request_body_filter(ngx_http_request_t *r, ngx_chain_t *in) /* rc == NGX_OK */ if (max != -1 && (uint64_t) (max - rb->received) < st->length) { + + if (r->headers_in.content_length_n != -1) { + ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, + "client intended to send body data " + "larger than declared"); + + return NGX_HTTP_BAD_REQUEST; + } + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, "client intended to send too large " "body: %O+%ui bytes", From 6f7494081ae8a56664afb480eff583d639b60ab4 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Fri, 3 May 2024 20:29:01 +0400 Subject: [PATCH 126/279] SSL: fixed possible configuration overwrite loading "engine:" keys. When loading certificate keys via ENGINE_load_private_key() in runtime, it was possible to overwrite configuration on ENGINE_by_id() failure. OpenSSL documention doesn't describe errors in details, the only reason I found in the comment to example is when the engine is not available. --- src/event/ngx_event_openssl.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c index 89f277fe56a..8d1f5695cbd 100644 --- a/src/event/ngx_event_openssl.c +++ b/src/event/ngx_event_openssl.c @@ -764,13 +764,13 @@ ngx_ssl_load_certificate_key(ngx_pool_t *pool, char **err, engine = ENGINE_by_id((char *) p); + *last++ = ':'; + if (engine == NULL) { *err = "ENGINE_by_id() failed"; return NULL; } - *last++ = ':'; - pkey = ENGINE_load_private_key(engine, (char *) last, 0, 0); if (pkey == NULL) { From 489e1e61912a808fdaffb4f513426cb285f267a3 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Fri, 3 May 2024 20:26:05 +0400 Subject: [PATCH 127/279] Stream pass: disabled passing from or to udp. Passing from udp was not possible for the most part due to preread buffer restriction. Passing to udp could occasionally work, but the connection would still be bound to the original listen rbtree, which prevented it from being deleted on connection closure. --- src/stream/ngx_stream_pass_module.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/stream/ngx_stream_pass_module.c b/src/stream/ngx_stream_pass_module.c index 2c1c60c6a83..1d671087c33 100644 --- a/src/stream/ngx_stream_pass_module.c +++ b/src/stream/ngx_stream_pass_module.c @@ -83,6 +83,11 @@ ngx_stream_pass_handler(ngx_stream_session_t *s) c->log->action = "passing connection to port"; + if (c->type == SOCK_DGRAM) { + ngx_log_error(NGX_LOG_ERR, c->log, 0, "cannot pass udp connection"); + goto failed; + } + if (c->buffer && c->buffer->pos != c->buffer->last) { ngx_log_error(NGX_LOG_ERR, c->log, 0, "cannot pass connection with preread data"); @@ -217,6 +222,10 @@ ngx_stream_pass_cleanup(void *data) static ngx_int_t ngx_stream_pass_match(ngx_listening_t *ls, ngx_addr_t *addr) { + if (ls->type == SOCK_DGRAM) { + return NGX_DECLINED; + } + if (!ls->wildcard) { return ngx_cmp_sockaddr(ls->sockaddr, ls->socklen, addr->sockaddr, addr->socklen, 1); From efc6a217b92985a1ee211b6bb7337cd2f62deb90 Mon Sep 17 00:00:00 2001 From: Edgar Bonet Date: Thu, 16 May 2024 11:15:10 +0200 Subject: [PATCH 128/279] Configure: fixed building libatomic test. Using "long *" instead of "AO_t *" leads either to -Wincompatible-pointer-types or -Wpointer-sign warnings, depending on whether long and size_t are compatible types (e.g., ILP32 versus LP64 data models). Notably, -Wpointer-sign warnings are enabled by default in Clang only, and -Wincompatible-pointer-types is an error starting from GCC 14. Signed-off-by: Edgar Bonet --- auto/lib/libatomic/conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auto/lib/libatomic/conf b/auto/lib/libatomic/conf index d1e484ab32e..8c8cb438b82 100644 --- a/auto/lib/libatomic/conf +++ b/auto/lib/libatomic/conf @@ -19,7 +19,7 @@ else #include " ngx_feature_path= ngx_feature_libs="-latomic_ops" - ngx_feature_test="long n = 0; + ngx_feature_test="AO_t n = 0; if (!AO_compare_and_swap(&n, 0, 1)) return 1; if (AO_fetch_and_add(&n, 1) != 1) From ea8270c6142869367c5608bff92df9f5b3f32d37 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Thu, 23 May 2024 19:15:38 +0400 Subject: [PATCH 129/279] Optimized chain link usage (ticket #2614). Previously chain links could sometimes be dropped instead of being reused, which could result in increased memory consumption during long requests. A similar chain link issue in ngx_http_gzip_filter_module was fixed in da46bfc484ef (1.11.10). Based on a patch by Sangmin Lee. --- src/core/ngx_output_chain.c | 10 ++++++++-- src/http/modules/ngx_http_grpc_module.c | 5 ++++- .../modules/ngx_http_gunzip_filter_module.c | 18 ++++++++++++++---- src/http/modules/ngx_http_gzip_filter_module.c | 10 +++++++--- src/http/modules/ngx_http_ssi_filter_module.c | 8 ++++++-- src/http/modules/ngx_http_sub_filter_module.c | 8 ++++++-- 6 files changed, 45 insertions(+), 14 deletions(-) diff --git a/src/core/ngx_output_chain.c b/src/core/ngx_output_chain.c index 85707425397..a46209c1700 100644 --- a/src/core/ngx_output_chain.c +++ b/src/core/ngx_output_chain.c @@ -117,7 +117,10 @@ ngx_output_chain(ngx_output_chain_ctx_t *ctx, ngx_chain_t *in) ngx_debug_point(); - ctx->in = ctx->in->next; + cl = ctx->in; + ctx->in = cl->next; + + ngx_free_chain(ctx->pool, cl); continue; } @@ -203,7 +206,10 @@ ngx_output_chain(ngx_output_chain_ctx_t *ctx, ngx_chain_t *in) /* delete the completed buf from the ctx->in chain */ if (ngx_buf_size(ctx->in->buf) == 0) { - ctx->in = ctx->in->next; + cl = ctx->in; + ctx->in = cl->next; + + ngx_free_chain(ctx->pool, cl); } cl = ngx_alloc_chain_link(ctx->pool); diff --git a/src/http/modules/ngx_http_grpc_module.c b/src/http/modules/ngx_http_grpc_module.c index dfe49c58618..e7726f3142d 100644 --- a/src/http/modules/ngx_http_grpc_module.c +++ b/src/http/modules/ngx_http_grpc_module.c @@ -1231,7 +1231,7 @@ ngx_http_grpc_body_output_filter(void *data, ngx_chain_t *in) ngx_buf_t *b; ngx_int_t rc; ngx_uint_t next, last; - ngx_chain_t *cl, *out, **ll; + ngx_chain_t *cl, *out, *ln, **ll; ngx_http_upstream_t *u; ngx_http_grpc_ctx_t *ctx; ngx_http_grpc_frame_t *f; @@ -1459,7 +1459,10 @@ ngx_http_grpc_body_output_filter(void *data, ngx_chain_t *in) last = 1; } + ln = in; in = in->next; + + ngx_free_chain(r->pool, ln); } ctx->in = in; diff --git a/src/http/modules/ngx_http_gunzip_filter_module.c b/src/http/modules/ngx_http_gunzip_filter_module.c index c1341f56285..5d170a1ba42 100644 --- a/src/http/modules/ngx_http_gunzip_filter_module.c +++ b/src/http/modules/ngx_http_gunzip_filter_module.c @@ -333,6 +333,8 @@ static ngx_int_t ngx_http_gunzip_filter_add_data(ngx_http_request_t *r, ngx_http_gunzip_ctx_t *ctx) { + ngx_chain_t *cl; + if (ctx->zstream.avail_in || ctx->flush != Z_NO_FLUSH || ctx->redo) { return NGX_OK; } @@ -344,8 +346,11 @@ ngx_http_gunzip_filter_add_data(ngx_http_request_t *r, return NGX_DECLINED; } - ctx->in_buf = ctx->in->buf; - ctx->in = ctx->in->next; + cl = ctx->in; + ctx->in_buf = cl->buf; + ctx->in = cl->next; + + ngx_free_chain(r->pool, cl); ctx->zstream.next_in = ctx->in_buf->pos; ctx->zstream.avail_in = ctx->in_buf->last - ctx->in_buf->pos; @@ -374,6 +379,7 @@ static ngx_int_t ngx_http_gunzip_filter_get_buf(ngx_http_request_t *r, ngx_http_gunzip_ctx_t *ctx) { + ngx_chain_t *cl; ngx_http_gunzip_conf_t *conf; if (ctx->zstream.avail_out) { @@ -383,8 +389,12 @@ ngx_http_gunzip_filter_get_buf(ngx_http_request_t *r, conf = ngx_http_get_module_loc_conf(r, ngx_http_gunzip_filter_module); if (ctx->free) { - ctx->out_buf = ctx->free->buf; - ctx->free = ctx->free->next; + + cl = ctx->free; + ctx->out_buf = cl->buf; + ctx->free = cl->next; + + ngx_free_chain(r->pool, cl); ctx->out_buf->flush = 0; diff --git a/src/http/modules/ngx_http_gzip_filter_module.c b/src/http/modules/ngx_http_gzip_filter_module.c index ed0de609aaa..b555278454c 100644 --- a/src/http/modules/ngx_http_gzip_filter_module.c +++ b/src/http/modules/ngx_http_gzip_filter_module.c @@ -985,10 +985,14 @@ static void ngx_http_gzip_filter_free_copy_buf(ngx_http_request_t *r, ngx_http_gzip_ctx_t *ctx) { - ngx_chain_t *cl; + ngx_chain_t *cl, *ln; + + for (cl = ctx->copied; cl; /* void */) { + ln = cl; + cl = cl->next; - for (cl = ctx->copied; cl; cl = cl->next) { - ngx_pfree(r->pool, cl->buf->start); + ngx_pfree(r->pool, ln->buf->start); + ngx_free_chain(r->pool, ln); } ctx->copied = NULL; diff --git a/src/http/modules/ngx_http_ssi_filter_module.c b/src/http/modules/ngx_http_ssi_filter_module.c index 0b84bd32249..47068f7550e 100644 --- a/src/http/modules/ngx_http_ssi_filter_module.c +++ b/src/http/modules/ngx_http_ssi_filter_module.c @@ -482,9 +482,13 @@ ngx_http_ssi_body_filter(ngx_http_request_t *r, ngx_chain_t *in) while (ctx->in || ctx->buf) { if (ctx->buf == NULL) { - ctx->buf = ctx->in->buf; - ctx->in = ctx->in->next; + + cl = ctx->in; + ctx->buf = cl->buf; + ctx->in = cl->next; ctx->pos = ctx->buf->pos; + + ngx_free_chain(r->pool, cl); } if (ctx->state == ssi_start_state) { diff --git a/src/http/modules/ngx_http_sub_filter_module.c b/src/http/modules/ngx_http_sub_filter_module.c index 6d3de59b09a..456bb27e304 100644 --- a/src/http/modules/ngx_http_sub_filter_module.c +++ b/src/http/modules/ngx_http_sub_filter_module.c @@ -335,9 +335,13 @@ ngx_http_sub_body_filter(ngx_http_request_t *r, ngx_chain_t *in) while (ctx->in || ctx->buf) { if (ctx->buf == NULL) { - ctx->buf = ctx->in->buf; - ctx->in = ctx->in->next; + + cl = ctx->in; + ctx->buf = cl->buf; + ctx->in = cl->next; ctx->pos = ctx->buf->pos; + + ngx_free_chain(r->pool, cl); } if (ctx->buf->flush || ctx->buf->recycled) { From 71ca978a352e025151a78bfcedc0d64814b062cb Mon Sep 17 00:00:00 2001 From: J Carter Date: Sat, 25 Nov 2023 21:57:09 +0000 Subject: [PATCH 130/279] Upstream: variables support in proxy_limit_rate and friends. --- src/http/modules/ngx_http_fastcgi_module.c | 8 ++++---- src/http/modules/ngx_http_proxy_module.c | 8 ++++---- src/http/modules/ngx_http_scgi_module.c | 8 ++++---- src/http/modules/ngx_http_uwsgi_module.c | 8 ++++---- src/http/ngx_http_upstream.c | 2 +- src/http/ngx_http_upstream.h | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/http/modules/ngx_http_fastcgi_module.c b/src/http/modules/ngx_http_fastcgi_module.c index b9890833d98..46a56f54e9f 100644 --- a/src/http/modules/ngx_http_fastcgi_module.c +++ b/src/http/modules/ngx_http_fastcgi_module.c @@ -375,7 +375,7 @@ static ngx_command_t ngx_http_fastcgi_commands[] = { { ngx_string("fastcgi_limit_rate"), NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, - ngx_conf_set_size_slot, + ngx_http_set_complex_value_size_slot, NGX_HTTP_LOC_CONF_OFFSET, offsetof(ngx_http_fastcgi_loc_conf_t, upstream.limit_rate), NULL }, @@ -2898,7 +2898,7 @@ ngx_http_fastcgi_create_loc_conf(ngx_conf_t *cf) conf->upstream.send_lowat = NGX_CONF_UNSET_SIZE; conf->upstream.buffer_size = NGX_CONF_UNSET_SIZE; - conf->upstream.limit_rate = NGX_CONF_UNSET_SIZE; + conf->upstream.limit_rate = NGX_CONF_UNSET_PTR; conf->upstream.busy_buffers_size_conf = NGX_CONF_UNSET_SIZE; conf->upstream.max_temp_file_size_conf = NGX_CONF_UNSET_SIZE; @@ -3015,8 +3015,8 @@ ngx_http_fastcgi_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) prev->upstream.buffer_size, (size_t) ngx_pagesize); - ngx_conf_merge_size_value(conf->upstream.limit_rate, - prev->upstream.limit_rate, 0); + ngx_conf_merge_ptr_value(conf->upstream.limit_rate, + prev->upstream.limit_rate, NULL); ngx_conf_merge_bufs_value(conf->upstream.bufs, prev->upstream.bufs, diff --git a/src/http/modules/ngx_http_proxy_module.c b/src/http/modules/ngx_http_proxy_module.c index 9cc202c9d08..b774c866e27 100644 --- a/src/http/modules/ngx_http_proxy_module.c +++ b/src/http/modules/ngx_http_proxy_module.c @@ -494,7 +494,7 @@ static ngx_command_t ngx_http_proxy_commands[] = { { ngx_string("proxy_limit_rate"), NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, - ngx_conf_set_size_slot, + ngx_http_set_complex_value_size_slot, NGX_HTTP_LOC_CONF_OFFSET, offsetof(ngx_http_proxy_loc_conf_t, upstream.limit_rate), NULL }, @@ -3371,7 +3371,7 @@ ngx_http_proxy_create_loc_conf(ngx_conf_t *cf) conf->upstream.send_lowat = NGX_CONF_UNSET_SIZE; conf->upstream.buffer_size = NGX_CONF_UNSET_SIZE; - conf->upstream.limit_rate = NGX_CONF_UNSET_SIZE; + conf->upstream.limit_rate = NGX_CONF_UNSET_PTR; conf->upstream.busy_buffers_size_conf = NGX_CONF_UNSET_SIZE; conf->upstream.max_temp_file_size_conf = NGX_CONF_UNSET_SIZE; @@ -3515,8 +3515,8 @@ ngx_http_proxy_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) prev->upstream.buffer_size, (size_t) ngx_pagesize); - ngx_conf_merge_size_value(conf->upstream.limit_rate, - prev->upstream.limit_rate, 0); + ngx_conf_merge_ptr_value(conf->upstream.limit_rate, + prev->upstream.limit_rate, NULL); ngx_conf_merge_bufs_value(conf->upstream.bufs, prev->upstream.bufs, 8, ngx_pagesize); diff --git a/src/http/modules/ngx_http_scgi_module.c b/src/http/modules/ngx_http_scgi_module.c index 3acea87b764..f818fc4b03c 100644 --- a/src/http/modules/ngx_http_scgi_module.c +++ b/src/http/modules/ngx_http_scgi_module.c @@ -223,7 +223,7 @@ static ngx_command_t ngx_http_scgi_commands[] = { { ngx_string("scgi_limit_rate"), NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, - ngx_conf_set_size_slot, + ngx_http_set_complex_value_size_slot, NGX_HTTP_LOC_CONF_OFFSET, offsetof(ngx_http_scgi_loc_conf_t, upstream.limit_rate), NULL }, @@ -1301,7 +1301,7 @@ ngx_http_scgi_create_loc_conf(ngx_conf_t *cf) conf->upstream.send_lowat = NGX_CONF_UNSET_SIZE; conf->upstream.buffer_size = NGX_CONF_UNSET_SIZE; - conf->upstream.limit_rate = NGX_CONF_UNSET_SIZE; + conf->upstream.limit_rate = NGX_CONF_UNSET_PTR; conf->upstream.busy_buffers_size_conf = NGX_CONF_UNSET_SIZE; conf->upstream.max_temp_file_size_conf = NGX_CONF_UNSET_SIZE; @@ -1413,8 +1413,8 @@ ngx_http_scgi_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) prev->upstream.buffer_size, (size_t) ngx_pagesize); - ngx_conf_merge_size_value(conf->upstream.limit_rate, - prev->upstream.limit_rate, 0); + ngx_conf_merge_ptr_value(conf->upstream.limit_rate, + prev->upstream.limit_rate, NULL); ngx_conf_merge_bufs_value(conf->upstream.bufs, prev->upstream.bufs, diff --git a/src/http/modules/ngx_http_uwsgi_module.c b/src/http/modules/ngx_http_uwsgi_module.c index c1731ff484c..c965d4bbd15 100644 --- a/src/http/modules/ngx_http_uwsgi_module.c +++ b/src/http/modules/ngx_http_uwsgi_module.c @@ -289,7 +289,7 @@ static ngx_command_t ngx_http_uwsgi_commands[] = { { ngx_string("uwsgi_limit_rate"), NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, - ngx_conf_set_size_slot, + ngx_http_set_complex_value_size_slot, NGX_HTTP_LOC_CONF_OFFSET, offsetof(ngx_http_uwsgi_loc_conf_t, upstream.limit_rate), NULL }, @@ -1532,7 +1532,7 @@ ngx_http_uwsgi_create_loc_conf(ngx_conf_t *cf) conf->upstream.send_lowat = NGX_CONF_UNSET_SIZE; conf->upstream.buffer_size = NGX_CONF_UNSET_SIZE; - conf->upstream.limit_rate = NGX_CONF_UNSET_SIZE; + conf->upstream.limit_rate = NGX_CONF_UNSET_PTR; conf->upstream.busy_buffers_size_conf = NGX_CONF_UNSET_SIZE; conf->upstream.max_temp_file_size_conf = NGX_CONF_UNSET_SIZE; @@ -1656,8 +1656,8 @@ ngx_http_uwsgi_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) prev->upstream.buffer_size, (size_t) ngx_pagesize); - ngx_conf_merge_size_value(conf->upstream.limit_rate, - prev->upstream.limit_rate, 0); + ngx_conf_merge_ptr_value(conf->upstream.limit_rate, + prev->upstream.limit_rate, NULL); ngx_conf_merge_bufs_value(conf->upstream.bufs, prev->upstream.bufs, diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c index 2ce9f211441..d29e503cb78 100644 --- a/src/http/ngx_http_upstream.c +++ b/src/http/ngx_http_upstream.c @@ -3236,7 +3236,7 @@ ngx_http_upstream_send_response(ngx_http_request_t *r, ngx_http_upstream_t *u) p->downstream = c; p->pool = r->pool; p->log = c->log; - p->limit_rate = u->conf->limit_rate; + p->limit_rate = ngx_http_complex_value_size(r, u->conf->limit_rate, 0); p->start_sec = ngx_time(); p->cacheable = u->cacheable || u->store; diff --git a/src/http/ngx_http_upstream.h b/src/http/ngx_http_upstream.h index 9a17a03a88f..cb16f2b4d5b 100644 --- a/src/http/ngx_http_upstream.h +++ b/src/http/ngx_http_upstream.h @@ -156,7 +156,7 @@ typedef struct { size_t send_lowat; size_t buffer_size; - size_t limit_rate; + ngx_http_complex_value_t *limit_rate; size_t busy_buffers_size; size_t max_temp_file_size; From 683e304e8bfe881ef983a0f9ef5e724eec2bd974 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Tue, 28 May 2024 17:17:19 +0400 Subject: [PATCH 131/279] QUIC: client transport parameter data length checking. --- src/event/quic/ngx_event_quic_transport.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/event/quic/ngx_event_quic_transport.c b/src/event/quic/ngx_event_quic_transport.c index 19670a6b1dc..fba098caa95 100644 --- a/src/event/quic/ngx_event_quic_transport.c +++ b/src/event/quic/ngx_event_quic_transport.c @@ -1750,6 +1750,14 @@ ngx_quic_parse_transport_params(u_char *p, u_char *end, ngx_quic_tp_t *tp, return NGX_ERROR; } + if ((size_t) (end - p) < len) { + ngx_log_error(NGX_LOG_INFO, log, 0, + "quic failed to parse" + " transport param id:0x%xL, data length %uL too long", + id, len); + return NGX_ERROR; + } + rc = ngx_quic_parse_transport_param(p, p + len, id, tp); if (rc == NGX_ERROR) { From 0fd59c8b565c4577f7c25b9e6450bd311d18f5e2 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Tue, 28 May 2024 17:18:28 +0400 Subject: [PATCH 132/279] HTTP/3: decoder stream pre-creation. Previously a decoder stream was created on demand for sending Section Acknowledgement, Stream Cancellation and Insert Count Increment. If conditions for sending any of these instructions never happen, a decoder stream is not created at all. These conditions include client not using the dynamic table and no streams abandoned by server (RFC 9204, Section 2.2.2.2). However RFC 9204, Section 4.2 defines only one condition for not creating a decoder stream: An endpoint MAY avoid creating a decoder stream if its decoder sets the maximum capacity of the dynamic table to zero. The change enables pre-creation of the decoder stream at HTTP/3 session initialization if maximum dynamic table capacity is not zero. Note that this value is currently hardcoded to 4096 bytes and is not configurable, so the stream is now always created. Also, the change fixes a potential stack overflow when creating a decoder stream in ngx_http_v3_send_cancel_stream() while draining a request stream by ngx_drain_connections(). Creating a decoder stream involves calling ngx_get_connection(), which calls ngx_drain_connections(), which will drain the same request stream again. If client's MAX_STREAMS for uni stream is high enough, these recursive calls will continue until we run out of stack. Otherwise, decoder stream creation will fail at some point and the request stream connection will be drained. This may result in use-after-free, since this connection could still be referenced up the stack. --- src/http/v3/ngx_http_v3_request.c | 20 ++++++++++++++------ src/http/v3/ngx_http_v3_uni.c | 4 +--- src/http/v3/ngx_http_v3_uni.h | 2 ++ 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/src/http/v3/ngx_http_v3_request.c b/src/http/v3/ngx_http_v3_request.c index aec122be612..e41ad50a8c0 100644 --- a/src/http/v3/ngx_http_v3_request.c +++ b/src/http/v3/ngx_http_v3_request.c @@ -134,7 +134,17 @@ ngx_http_v3_init(ngx_connection_t *c) } } - return ngx_http_v3_send_settings(c); + if (ngx_http_v3_send_settings(c) != NGX_OK) { + return NGX_ERROR; + } + + if (h3scf->max_table_capacity > 0) { + if (ngx_http_v3_get_uni_stream(c, NGX_HTTP_V3_STREAM_DECODER) == NULL) { + return NGX_ERROR; + } + } + + return NGX_OK; } @@ -398,14 +408,12 @@ ngx_http_v3_wait_request_handler(ngx_event_t *rev) void ngx_http_v3_reset_stream(ngx_connection_t *c) { - ngx_http_v3_session_t *h3c; - ngx_http_v3_srv_conf_t *h3scf; - - h3scf = ngx_http_v3_get_module_srv_conf(c, ngx_http_v3_module); + ngx_http_v3_session_t *h3c; h3c = ngx_http_v3_get_session(c); - if (h3scf->max_table_capacity > 0 && !c->read->eof && !h3c->hq + if (!c->read->eof && !h3c->hq + && h3c->known_streams[NGX_HTTP_V3_STREAM_SERVER_DECODER] && (c->quic->id & NGX_QUIC_STREAM_UNIDIRECTIONAL) == 0) { (void) ngx_http_v3_send_cancel_stream(c, c->quic->id); diff --git a/src/http/v3/ngx_http_v3_uni.c b/src/http/v3/ngx_http_v3_uni.c index 2fc5b07a48f..302064b8b2b 100644 --- a/src/http/v3/ngx_http_v3_uni.c +++ b/src/http/v3/ngx_http_v3_uni.c @@ -20,8 +20,6 @@ static void ngx_http_v3_close_uni_stream(ngx_connection_t *c); static void ngx_http_v3_uni_read_handler(ngx_event_t *rev); static void ngx_http_v3_uni_dummy_read_handler(ngx_event_t *wev); static void ngx_http_v3_uni_dummy_write_handler(ngx_event_t *wev); -static ngx_connection_t *ngx_http_v3_get_uni_stream(ngx_connection_t *c, - ngx_uint_t type); void @@ -307,7 +305,7 @@ ngx_http_v3_uni_dummy_write_handler(ngx_event_t *wev) } -static ngx_connection_t * +ngx_connection_t * ngx_http_v3_get_uni_stream(ngx_connection_t *c, ngx_uint_t type) { u_char buf[NGX_HTTP_V3_VARLEN_INT_LEN]; diff --git a/src/http/v3/ngx_http_v3_uni.h b/src/http/v3/ngx_http_v3_uni.h index 911e153d793..2805894697a 100644 --- a/src/http/v3/ngx_http_v3_uni.h +++ b/src/http/v3/ngx_http_v3_uni.h @@ -19,6 +19,8 @@ ngx_int_t ngx_http_v3_register_uni_stream(ngx_connection_t *c, uint64_t type); ngx_int_t ngx_http_v3_cancel_stream(ngx_connection_t *c, ngx_uint_t stream_id); +ngx_connection_t *ngx_http_v3_get_uni_stream(ngx_connection_t *c, + ngx_uint_t type); ngx_int_t ngx_http_v3_send_settings(ngx_connection_t *c); ngx_int_t ngx_http_v3_send_goaway(ngx_connection_t *c, uint64_t id); ngx_int_t ngx_http_v3_send_ack_section(ngx_connection_t *c, From cca5655dd9ba349817946a0db14f8b1f633f700a Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Tue, 28 May 2024 17:18:50 +0400 Subject: [PATCH 133/279] HTTP/3: fixed dynamic table overflow. While inserting a new entry into the dynamic table, first the entry is added, and then older entries are evicted until table size is within capacity. After the first step, the number of entries may temporarily exceed the maximum calculated from capacity by one entry, which previously caused table overflow. The easiest way to trigger the issue is to keep adding entries with empty names and values until first eviction. The issue was introduced by 987bee4363d1. --- src/http/v3/ngx_http_v3_table.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/http/v3/ngx_http_v3_table.c b/src/http/v3/ngx_http_v3_table.c index f49a8fc5e2f..428e7326b91 100644 --- a/src/http/v3/ngx_http_v3_table.c +++ b/src/http/v3/ngx_http_v3_table.c @@ -308,7 +308,7 @@ ngx_http_v3_set_capacity(ngx_connection_t *c, ngx_uint_t capacity) prev_max = dt->capacity / 32; if (max > prev_max) { - elts = ngx_alloc(max * sizeof(void *), c->log); + elts = ngx_alloc((max + 1) * sizeof(void *), c->log); if (elts == NULL) { return NGX_ERROR; } From 6f8c520f497edfe26f46ef6da6699174df5b3da4 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Tue, 28 May 2024 17:19:08 +0400 Subject: [PATCH 134/279] QUIC: ignore CRYPTO frames after handshake completion. Sending handshake-level CRYPTO frames after the client's Finished message could lead to memory disclosure and a potential segfault, if those frames are sent in one packet with the Finished frame. --- src/event/quic/ngx_event_quic_ssl.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/event/quic/ngx_event_quic_ssl.c b/src/event/quic/ngx_event_quic_ssl.c index 7872783f898..ba0b5929fde 100644 --- a/src/event/quic/ngx_event_quic_ssl.c +++ b/src/event/quic/ngx_event_quic_ssl.c @@ -326,6 +326,11 @@ ngx_quic_handle_crypto_frame(ngx_connection_t *c, ngx_quic_header_t *pkt, ngx_quic_crypto_frame_t *f; qc = ngx_quic_get_connection(c); + + if (!ngx_quic_keys_available(qc->keys, pkt->level, 0)) { + return NGX_OK; + } + ctx = ngx_quic_get_send_ctx(qc, pkt->level); f = &frame->u.crypto; From 9ddc6a08f4c83db5acc0a74c223ddf75a44b726d Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Tue, 28 May 2024 17:19:21 +0400 Subject: [PATCH 135/279] QUIC: ngx_quic_buffer_t use-after-free protection. Previously the last chain field of ngx_quic_buffer_t could still reference freed chains and buffers after calling ngx_quic_free_buffer(). While normally an ngx_quic_buffer_t object should not be used after freeing, resetting last_chain field would prevent a potential use-after-free. --- src/event/quic/ngx_event_quic_frames.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/event/quic/ngx_event_quic_frames.c b/src/event/quic/ngx_event_quic_frames.c index 42b7d9f41de..6ea908cc1ca 100644 --- a/src/event/quic/ngx_event_quic_frames.c +++ b/src/event/quic/ngx_event_quic_frames.c @@ -648,6 +648,7 @@ ngx_quic_free_buffer(ngx_connection_t *c, ngx_quic_buffer_t *qb) ngx_quic_free_chain(c, qb->chain); qb->chain = NULL; + qb->last_chain = NULL; } From 34bd899287e2e00cbd00e2f60e5013b12e7b95b0 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Tue, 28 May 2024 17:20:45 +0400 Subject: [PATCH 136/279] HTTP/3: fixed handling of zero-length literal field line. Previously, st->value was passed with NULL data pointer to header handlers. --- src/http/v3/ngx_http_v3_parse.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/http/v3/ngx_http_v3_parse.c b/src/http/v3/ngx_http_v3_parse.c index 568816323e1..436765c8a40 100644 --- a/src/http/v3/ngx_http_v3_parse.c +++ b/src/http/v3/ngx_http_v3_parse.c @@ -810,6 +810,7 @@ ngx_http_v3_parse_field_lri(ngx_connection_t *c, st->literal.length = st->pint.value; if (st->literal.length == 0) { + st->value.data = (u_char *) ""; goto done; } @@ -932,6 +933,7 @@ ngx_http_v3_parse_field_l(ngx_connection_t *c, st->literal.length = st->pint.value; if (st->literal.length == 0) { + st->value.data = (u_char *) ""; goto done; } @@ -1072,6 +1074,7 @@ ngx_http_v3_parse_field_lpbi(ngx_connection_t *c, st->literal.length = st->pint.value; if (st->literal.length == 0) { + st->value.data = (u_char *) ""; goto done; } From 0ddcae05b6725f78352d7cbe4ba441d9a411e2cf Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Tue, 28 May 2024 17:19:38 +0400 Subject: [PATCH 137/279] nginx-1.27.0-RELEASE --- docs/xml/nginx/changes.xml | 68 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/docs/xml/nginx/changes.xml b/docs/xml/nginx/changes.xml index e71dd083326..984dd5ced8a 100644 --- a/docs/xml/nginx/changes.xml +++ b/docs/xml/nginx/changes.xml @@ -5,6 +5,74 @@ + + + + +при использовании HTTP/3 обработка специально созданной QUIC-сессии могла +приводить к падению рабочего процесса, отправке клиенту содержимого памяти +рабочего процесса на системах с MTU больше 4096 байт, а также потенциально +могла иметь другие последствия +(CVE-2024-32760, CVE-2024-31079, CVE-2024-35200, CVE-2024-34161).
+Спасибо Nils Bars из CISPA. + + +when using HTTP/3, processing of a specially crafted QUIC session might +cause a worker process crash, worker process memory disclosure on systems +with MTU larger than 4096 bytes, or might have potential other impact +(CVE-2024-32760, CVE-2024-31079, CVE-2024-35200, CVE-2024-34161).
+Thanks to Nils Bars of CISPA. + + + + + +директивы proxy_limit_rate, fastcgi_limit_rate, +scgi_limit_rate и uwsgi_limit_rate поддерживают переменные. + + +variables support +in the "proxy_limit_rate", "fastcgi_limit_rate", "scgi_limit_rate", +and "uwsgi_limit_rate" directives. + + + + + +уменьшено потребление памяти для долгоживущих запросов, +если используются директивы gzip, gunzip, ssi, sub_filter или grpc_pass. + + +reduced memory consumption for long-lived requests +if "gzip", "gunzip", "ssi", "sub_filter", or "grpc_pass" directives are used. + + + + + +nginx не собирался gcc 14, +если использовался параметр --with-atomic.
+Спасибо Edgar Bonet. + + +nginx could not be built by gcc 14 +if the --with-atomic option was used.
+Thanks to Edgar Bonet. + + + + + +Исправления в HTTP/3. + + +Bugfixes in HTTP/3. + + + + + + From e734df6664e70f118ca3140bcef6d4f1750fa8fa Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Tue, 28 May 2024 17:22:30 +0400 Subject: [PATCH 138/279] release-1.27.0 tag --- .hgtags | 1 + 1 file changed, 1 insertion(+) diff --git a/.hgtags b/.hgtags index 4a7ee26780a..0878e27f750 100644 --- a/.hgtags +++ b/.hgtags @@ -478,3 +478,4 @@ f8134640e8615448205785cf00b0bc810489b495 release-1.25.1 294a3d07234f8f65d7b0e0b0e2c5b05c12c5da0a release-1.25.3 173a0a7dbce569adbb70257c6ec4f0f6bc585009 release-1.25.4 8618e4d900cc71082fbe7dc72af087937d64faf5 release-1.25.5 +2166e329fb4ed7d6da7c823ee6499f7d06d7bc00 release-1.27.0 From 788e462c5b81f5f1aee475488e10f01680c530e9 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Thu, 27 Jun 2024 17:29:56 +0400 Subject: [PATCH 139/279] Stream: allow servers with no handler. Previously handlers were mandatory. However they are not always needed. For example, a server configured with ssl_reject_handshake does not need a handler. Such servers required a fake handler to pass the check. Now handler absence check is moved to runtime. If handler is missing, the connection is closed with 500 code. --- src/stream/ngx_stream_core_module.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/stream/ngx_stream_core_module.c b/src/stream/ngx_stream_core_module.c index 3093963b7df..40951c29146 100644 --- a/src/stream/ngx_stream_core_module.c +++ b/src/stream/ngx_stream_core_module.c @@ -458,6 +458,13 @@ ngx_stream_core_content_phase(ngx_stream_session_t *s, return NGX_OK; } + if (cscf->handler == NULL) { + ngx_log_debug0(NGX_LOG_DEBUG_STREAM, c->log, 0, + "no handler for server"); + ngx_stream_finalize_session(s, NGX_STREAM_INTERNAL_SERVER_ERROR); + return NGX_OK; + } + cscf->handler(s); return NGX_OK; @@ -734,13 +741,6 @@ ngx_stream_core_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) conf->resolver = prev->resolver; } - if (conf->handler == NULL) { - ngx_log_error(NGX_LOG_EMERG, cf->log, 0, - "no handler for server in %s:%ui", - conf->file_name, conf->line); - return NGX_CONF_ERROR; - } - if (conf->error_log == NULL) { if (prev->error_log) { conf->error_log = prev->error_log; From 145b228530c364452c14d3184f1eee5e09b324aa Mon Sep 17 00:00:00 2001 From: Kasei Wang Date: Thu, 18 Jul 2024 17:43:25 +0400 Subject: [PATCH 140/279] HTTP/2: close connections initialized during graceful shutdown. In some rare cases, graceful shutdown may happen while initializing an HTTP/2 connection. Previously, such a connection ignored the shutdown and remained active. Now it is gracefully closed prior to processing any streams to eliminate the shutdown delay. --- src/http/v2/ngx_http_v2.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/http/v2/ngx_http_v2.c b/src/http/v2/ngx_http_v2.c index 0f5bd3de8d0..91a28b2281b 100644 --- a/src/http/v2/ngx_http_v2.c +++ b/src/http/v2/ngx_http_v2.c @@ -292,6 +292,11 @@ ngx_http_v2_init(ngx_event_t *rev) c->data = h2c; + if (ngx_exiting) { + ngx_http_v2_finalize_connection(h2c, NGX_HTTP_V2_NO_ERROR); + return; + } + rev->handler = ngx_http_v2_read_handler; c->write->handler = ngx_http_v2_write_handler; From 6ecc4e38079de24475ad9d47162d8dabd4788e73 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Fri, 9 Aug 2024 18:01:42 +0400 Subject: [PATCH 141/279] Version bump. --- src/core/nginx.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/nginx.h b/src/core/nginx.h index d21b118eea3..b7d02cbceea 100644 --- a/src/core/nginx.h +++ b/src/core/nginx.h @@ -9,8 +9,8 @@ #define _NGINX_H_INCLUDED_ -#define nginx_version 1027000 -#define NGINX_VERSION "1.27.0" +#define nginx_version 1027001 +#define NGINX_VERSION "1.27.1" #define NGINX_VER "nginx/" NGINX_VERSION #ifdef NGX_BUILD From 250baed4ee9bd32b6f236fd9ec17bf8800839b0e Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Fri, 9 Aug 2024 19:12:23 +0400 Subject: [PATCH 142/279] Typo fixed. --- docs/xml/nginx/changes.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/xml/nginx/changes.xml b/docs/xml/nginx/changes.xml index 984dd5ced8a..c6729a0f39e 100644 --- a/docs/xml/nginx/changes.xml +++ b/docs/xml/nginx/changes.xml @@ -51,12 +51,12 @@ if "gzip", "gunzip", "ssi", "sub_filter", or "grpc_pass" directives are used. nginx не собирался gcc 14, -если использовался параметр --with-atomic.
+если использовался параметр --with-libatomic.
Спасибо Edgar Bonet. nginx could not be built by gcc 14 -if the --with-atomic option was used.
+if the --with-libatomic option was used.
Thanks to Edgar Bonet. From 58b92177e7c3c50f77f807ab3846ad5c7bbf0ebe Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Fri, 9 Aug 2024 19:12:25 +0400 Subject: [PATCH 143/279] QUIC: discarding 0-RTT keys. For simplicity, this is done on successful decryption of a 1-RTT packet. --- src/event/quic/ngx_event_quic.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/event/quic/ngx_event_quic.c b/src/event/quic/ngx_event_quic.c index e4690f7dddc..c03b1d003ad 100644 --- a/src/event/quic/ngx_event_quic.c +++ b/src/event/quic/ngx_event_quic.c @@ -1022,6 +1022,16 @@ ngx_quic_handle_payload(ngx_connection_t *c, ngx_quic_header_t *pkt) } } + if (pkt->level == ssl_encryption_application) { + /* + * RFC 9001, 4.9.3. Discarding 0-RTT Keys + * + * After receiving a 1-RTT packet, servers MUST discard + * 0-RTT keys within a short time + */ + ngx_quic_discard_ctx(c, ssl_encryption_early_data); + } + if (qc->closing) { /* * RFC 9000, 10.2. Immediate Close From 504c78fc6dd9542371b1658c9c8fdac6be20d2f6 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Fri, 9 Aug 2024 19:12:26 +0400 Subject: [PATCH 144/279] QUIC: zero out existing keying material only. Previously, this used to have extra ngx_explicit_memzero() calls from within ngx_quic_keys_cleanup(), which might be suboptimal. --- src/event/quic/ngx_event_quic_protection.c | 29 +++++++++++++++++----- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/src/event/quic/ngx_event_quic_protection.c b/src/event/quic/ngx_event_quic_protection.c index 8223626b65f..55f0f6fd70f 100644 --- a/src/event/quic/ngx_event_quic_protection.c +++ b/src/event/quic/ngx_event_quic_protection.c @@ -743,8 +743,15 @@ ngx_quic_keys_discard(ngx_quic_keys_t *keys, ngx_quic_crypto_hp_cleanup(client); ngx_quic_crypto_hp_cleanup(server); - ngx_explicit_memzero(client->secret.data, client->secret.len); - ngx_explicit_memzero(server->secret.data, server->secret.len); + if (client->secret.len) { + ngx_explicit_memzero(client->secret.data, client->secret.len); + client->secret.len = 0; + } + + if (server->secret.len) { + ngx_explicit_memzero(server->secret.data, server->secret.len); + server->secret.len = 0; + } } @@ -844,6 +851,9 @@ ngx_quic_keys_update(ngx_event_t *ev) ngx_explicit_memzero(current->server.secret.data, current->server.secret.len); + current->client.secret.len = 0; + current->server.secret.len = 0; + ngx_explicit_memzero(client_key.data, client_key.len); ngx_explicit_memzero(server_key.data, server_key.len); @@ -870,10 +880,17 @@ ngx_quic_keys_cleanup(ngx_quic_keys_t *keys) ngx_quic_crypto_cleanup(&next->client); ngx_quic_crypto_cleanup(&next->server); - ngx_explicit_memzero(next->client.secret.data, - next->client.secret.len); - ngx_explicit_memzero(next->server.secret.data, - next->server.secret.len); + if (next->client.secret.len) { + ngx_explicit_memzero(next->client.secret.data, + next->client.secret.len); + next->client.secret.len = 0; + } + + if (next->server.secret.len) { + ngx_explicit_memzero(next->server.secret.data, + next->server.secret.len); + next->server.secret.len = 0; + } } From 0fa8434957dcecef934a70e9c92d40a0a08988bd Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Fri, 9 Aug 2024 19:12:26 +0400 Subject: [PATCH 145/279] Stream ssl_preread: do not reallocate a parsed SNI host. We own this memory from the session pool. --- src/stream/ngx_stream_ssl_preread_module.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/stream/ngx_stream_ssl_preread_module.c b/src/stream/ngx_stream_ssl_preread_module.c index bc96adeee7f..3fc83ff2f4c 100644 --- a/src/stream/ngx_stream_ssl_preread_module.c +++ b/src/stream/ngx_stream_ssl_preread_module.c @@ -519,7 +519,7 @@ ngx_stream_ssl_preread_servername(ngx_stream_session_t *s, host = *servername; - rc = ngx_stream_validate_host(&host, c->pool, 1); + rc = ngx_stream_validate_host(&host, c->pool, 0); if (rc == NGX_ERROR) { return NGX_ERROR; From 7362d01658b61184108c21278443910da68f93b4 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Mon, 12 Aug 2024 18:20:43 +0400 Subject: [PATCH 146/279] Mp4: fixed buffer underread while updating stsz atom. While cropping an stsc atom in ngx_http_mp4_crop_stsc_data(), a 32-bit integer overflow could happen, which could result in incorrect seeking and a very large value stored in "samples". This resulted in a large invalid value of trak->end_chunk_samples. This value is further used to calculate the value of trak->end_chunk_samples_size in ngx_http_mp4_update_stsz_atom(). While doing this, a large invalid value of trak->end_chunk_samples could result in reading memory before stsz atom start. This could potentially result in a segfault. --- src/http/modules/ngx_http_mp4_module.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/http/modules/ngx_http_mp4_module.c b/src/http/modules/ngx_http_mp4_module.c index 03175dea214..1cd017c274e 100644 --- a/src/http/modules/ngx_http_mp4_module.c +++ b/src/http/modules/ngx_http_mp4_module.c @@ -3099,7 +3099,8 @@ static ngx_int_t ngx_http_mp4_crop_stsc_data(ngx_http_mp4_file_t *mp4, ngx_http_mp4_trak_t *trak, ngx_uint_t start) { - uint32_t start_sample, chunk, samples, id, next_chunk, n, + uint64_t n; + uint32_t start_sample, chunk, samples, id, next_chunk, prev_samples; ngx_buf_t *data, *buf; ngx_uint_t entries, target_chunk, chunk_samples; @@ -3160,7 +3161,7 @@ ngx_http_mp4_crop_stsc_data(ngx_http_mp4_file_t *mp4, "samples:%uD, id:%uD", start_sample, chunk, next_chunk - chunk, samples, id); - n = (next_chunk - chunk) * samples; + n = (uint64_t) (next_chunk - chunk) * samples; if (start_sample < n) { goto found; @@ -3182,7 +3183,7 @@ ngx_http_mp4_crop_stsc_data(ngx_http_mp4_file_t *mp4, "sample:%uD, chunk:%uD, chunks:%uD, samples:%uD", start_sample, chunk, next_chunk - chunk, samples); - n = (next_chunk - chunk) * samples; + n = (uint64_t) (next_chunk - chunk) * samples; if (start_sample > n) { ngx_log_error(NGX_LOG_ERR, mp4->file.log, 0, From 88955b1044ef38315b77ad1a509d63631a790a0f Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Mon, 12 Aug 2024 18:20:45 +0400 Subject: [PATCH 147/279] Mp4: rejecting unordered chunks in stsc atom. Unordered chunks could result in trak->end_chunk smaller than trak->start_chunk in ngx_http_mp4_crop_stsc_data(). Later in ngx_http_mp4_update_stco_atom() this caused buffer overread while trying to calculate trak->end_offset. --- src/http/modules/ngx_http_mp4_module.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/http/modules/ngx_http_mp4_module.c b/src/http/modules/ngx_http_mp4_module.c index 1cd017c274e..041ad263b56 100644 --- a/src/http/modules/ngx_http_mp4_module.c +++ b/src/http/modules/ngx_http_mp4_module.c @@ -3156,6 +3156,13 @@ ngx_http_mp4_crop_stsc_data(ngx_http_mp4_file_t *mp4, next_chunk = ngx_mp4_get_32value(entry->chunk); + if (next_chunk < chunk) { + ngx_log_error(NGX_LOG_ERR, mp4->file.log, 0, + "unordered mp4 stsc chunks in \"%s\"", + mp4->file.name.data); + return NGX_ERROR; + } + ngx_log_debug5(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, "sample:%uD, chunk:%uD, chunks:%uD, " "samples:%uD, id:%uD", From c165589d098e08b5084d334be4612738f2bcfa84 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Mon, 12 Aug 2024 18:20:49 +0400 Subject: [PATCH 148/279] Updated OpenSSL used for win32 builds. --- misc/GNUmakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/GNUmakefile b/misc/GNUmakefile index dca0e263750..fa4c36d49f0 100644 --- a/misc/GNUmakefile +++ b/misc/GNUmakefile @@ -6,7 +6,7 @@ TEMP = tmp CC = cl OBJS = objs.msvc8 -OPENSSL = openssl-3.0.13 +OPENSSL = openssl-3.0.14 ZLIB = zlib-1.3.1 PCRE = pcre2-10.39 From e06bdbd4a20912c5223d7c6c6e2b3f0d6086c928 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Mon, 12 Aug 2024 18:20:52 +0400 Subject: [PATCH 149/279] nginx-1.27.1-RELEASE --- docs/xml/nginx/changes.xml | 49 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/docs/xml/nginx/changes.xml b/docs/xml/nginx/changes.xml index c6729a0f39e..9c2f317bc25 100644 --- a/docs/xml/nginx/changes.xml +++ b/docs/xml/nginx/changes.xml @@ -5,6 +5,55 @@ + + + + +обработка специально созданного mp4-файла модулем ngx_http_mp4_module +могла приводить к падению рабочего процесса (CVE-2024-7347).
+Спасибо Nils Bars. + + +processing of a specially crafted mp4 file by the ngx_http_mp4_module +might cause a worker process crash (CVE-2024-7347).
+Thanks to Nils Bars. + + + + + +теперь обработчик в модуле stream не является обязательным. + + +now the stream module handler is not mandatory. + + + + + +новые HTTP/2-соединения могли игнорировать +плавное завершение старых рабочих процессов.
+Спасибо Kasei Wang. + + +new HTTP/2 connections might ignore +graceful shutdown of old worker processes.
+Thanks to Kasei Wang. + + + + + +Исправления в HTTP/3. + + +Bugfixes in HTTP/3. + + + + + + From a4100450c067009158299ad69adb7d6f5e775943 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Mon, 12 Aug 2024 18:21:01 +0400 Subject: [PATCH 150/279] release-1.27.1 tag --- .hgtags | 1 + 1 file changed, 1 insertion(+) diff --git a/.hgtags b/.hgtags index 0878e27f750..2c9cb3bc6bc 100644 --- a/.hgtags +++ b/.hgtags @@ -479,3 +479,4 @@ f8134640e8615448205785cf00b0bc810489b495 release-1.25.1 173a0a7dbce569adbb70257c6ec4f0f6bc585009 release-1.25.4 8618e4d900cc71082fbe7dc72af087937d64faf5 release-1.25.5 2166e329fb4ed7d6da7c823ee6499f7d06d7bc00 release-1.27.0 +417b1045b18ecbca0ca583073a221ffbfbaee5d9 release-1.27.1 From 48ac1ee9c6a80a183f8cf61915fe1ab5047a43fa Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Tue, 20 Aug 2024 21:18:30 +0400 Subject: [PATCH 151/279] Version bump. --- src/core/nginx.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/nginx.h b/src/core/nginx.h index b7d02cbceea..140a2940874 100644 --- a/src/core/nginx.h +++ b/src/core/nginx.h @@ -9,8 +9,8 @@ #define _NGINX_H_INCLUDED_ -#define nginx_version 1027001 -#define NGINX_VERSION "1.27.1" +#define nginx_version 1027002 +#define NGINX_VERSION "1.27.2" #define NGINX_VER "nginx/" NGINX_VERSION #ifdef NGX_BUILD From 581cf22673fc63e206693294161ea32e691a432f Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Thu, 22 Aug 2024 14:57:45 +0400 Subject: [PATCH 152/279] Stream: client certificate validation with OCSP. --- src/stream/ngx_stream_ssl_module.c | 155 +++++++++++++++++++++++++++++ src/stream/ngx_stream_ssl_module.h | 4 + 2 files changed, 159 insertions(+) diff --git a/src/stream/ngx_stream_ssl_module.c b/src/stream/ngx_stream_ssl_module.c index ba444776a9d..37f9c547d6c 100644 --- a/src/stream/ngx_stream_ssl_module.c +++ b/src/stream/ngx_stream_ssl_module.c @@ -51,6 +51,8 @@ static char *ngx_stream_ssl_password_file(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); static char *ngx_stream_ssl_session_cache(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); +static char *ngx_stream_ssl_ocsp_cache(ngx_conf_t *cf, ngx_command_t *cmd, + void *conf); static char *ngx_stream_ssl_alpn(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); @@ -80,6 +82,14 @@ static ngx_conf_enum_t ngx_stream_ssl_verify[] = { }; +static ngx_conf_enum_t ngx_stream_ssl_ocsp[] = { + { ngx_string("off"), 0 }, + { ngx_string("on"), 1 }, + { ngx_string("leaf"), 2 }, + { ngx_null_string, 0 } +}; + + static ngx_conf_post_t ngx_stream_ssl_conf_command_post = { ngx_stream_ssl_conf_command_check }; @@ -212,6 +222,27 @@ static ngx_command_t ngx_stream_ssl_commands[] = { offsetof(ngx_stream_ssl_srv_conf_t, crl), NULL }, + { ngx_string("ssl_ocsp"), + NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_FLAG, + ngx_conf_set_enum_slot, + NGX_STREAM_SRV_CONF_OFFSET, + offsetof(ngx_stream_ssl_srv_conf_t, ocsp), + &ngx_stream_ssl_ocsp }, + + { ngx_string("ssl_ocsp_responder"), + NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_TAKE1, + ngx_conf_set_str_slot, + NGX_STREAM_SRV_CONF_OFFSET, + offsetof(ngx_stream_ssl_srv_conf_t, ocsp_responder), + NULL }, + + { ngx_string("ssl_ocsp_cache"), + NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_TAKE1, + ngx_stream_ssl_ocsp_cache, + NGX_STREAM_SRV_CONF_OFFSET, + 0, + NULL }, + { ngx_string("ssl_conf_command"), NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_TAKE2, ngx_conf_set_keyval_slot, @@ -777,6 +808,7 @@ ngx_stream_ssl_create_srv_conf(ngx_conf_t *cf) * sscf->alpn = { 0, NULL }; * sscf->ciphers = { 0, NULL }; * sscf->shm_zone = NULL; + * sscf->ocsp_responder = { 0, NULL }; */ sscf->handshake_timeout = NGX_CONF_UNSET_MSEC; @@ -792,6 +824,8 @@ ngx_stream_ssl_create_srv_conf(ngx_conf_t *cf) sscf->session_timeout = NGX_CONF_UNSET; sscf->session_tickets = NGX_CONF_UNSET; sscf->session_ticket_keys = NGX_CONF_UNSET_PTR; + sscf->ocsp = NGX_CONF_UNSET_UINT; + sscf->ocsp_cache_zone = NGX_CONF_UNSET_PTR; return sscf; } @@ -846,6 +880,10 @@ ngx_stream_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) ngx_conf_merge_ptr_value(conf->conf_commands, prev->conf_commands, NULL); + ngx_conf_merge_uint_value(conf->ocsp, prev->ocsp, 0); + ngx_conf_merge_str_value(conf->ocsp_responder, prev->ocsp_responder, ""); + ngx_conf_merge_ptr_value(conf->ocsp_cache_zone, + prev->ocsp_cache_zone, NULL); conf->ssl.log = cf->log; @@ -959,6 +997,23 @@ ngx_stream_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) } } + if (conf->ocsp) { + + if (conf->verify == 3) { + ngx_log_error(NGX_LOG_EMERG, cf->log, 0, + "\"ssl_ocsp\" is incompatible with " + "\"ssl_verify_client optional_no_ca\""); + return NGX_CONF_ERROR; + } + + if (ngx_ssl_ocsp(cf, &conf->ssl, &conf->ocsp_responder, conf->ocsp, + conf->ocsp_cache_zone) + != NGX_OK) + { + return NGX_CONF_ERROR; + } + } + if (ngx_ssl_dhparam(cf, &conf->ssl, &conf->dhparam) != NGX_OK) { return NGX_CONF_ERROR; } @@ -1231,6 +1286,85 @@ ngx_stream_ssl_session_cache(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) } +static char * +ngx_stream_ssl_ocsp_cache(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) +{ + ngx_stream_ssl_srv_conf_t *sscf = conf; + + size_t len; + ngx_int_t n; + ngx_str_t *value, name, size; + ngx_uint_t j; + + if (sscf->ocsp_cache_zone != NGX_CONF_UNSET_PTR) { + return "is duplicate"; + } + + value = cf->args->elts; + + if (ngx_strcmp(value[1].data, "off") == 0) { + sscf->ocsp_cache_zone = NULL; + return NGX_CONF_OK; + } + + if (value[1].len <= sizeof("shared:") - 1 + || ngx_strncmp(value[1].data, "shared:", sizeof("shared:") - 1) != 0) + { + goto invalid; + } + + len = 0; + + for (j = sizeof("shared:") - 1; j < value[1].len; j++) { + if (value[1].data[j] == ':') { + break; + } + + len++; + } + + if (len == 0 || j == value[1].len) { + goto invalid; + } + + name.len = len; + name.data = value[1].data + sizeof("shared:") - 1; + + size.len = value[1].len - j - 1; + size.data = name.data + len + 1; + + n = ngx_parse_size(&size); + + if (n == NGX_ERROR) { + goto invalid; + } + + if (n < (ngx_int_t) (8 * ngx_pagesize)) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "OCSP cache \"%V\" is too small", &value[1]); + + return NGX_CONF_ERROR; + } + + sscf->ocsp_cache_zone = ngx_shared_memory_add(cf, &name, n, + &ngx_stream_ssl_module_ctx); + if (sscf->ocsp_cache_zone == NULL) { + return NGX_CONF_ERROR; + } + + sscf->ocsp_cache_zone->init = ngx_ssl_ocsp_cache_init; + + return NGX_CONF_OK; + +invalid: + + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "invalid OCSP cache \"%V\"", &value[1]); + + return NGX_CONF_ERROR; +} + + static char * ngx_stream_ssl_alpn(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) { @@ -1308,6 +1442,27 @@ ngx_stream_ssl_init(ngx_conf_t *cf) ngx_stream_core_main_conf_t *cmcf; cmcf = ngx_stream_conf_get_module_main_conf(cf, ngx_stream_core_module); + cscfp = cmcf->servers.elts; + + for (s = 0; s < cmcf->servers.nelts; s++) { + + sscf = cscfp[s]->ctx->srv_conf[ngx_stream_ssl_module.ctx_index]; + + if (sscf->ssl.ctx == NULL) { + continue; + } + + cscf = cscfp[s]->ctx->srv_conf[ngx_stream_core_module.ctx_index]; + + if (sscf->ocsp) { + if (ngx_ssl_ocsp_resolver(cf, &sscf->ssl, cscf->resolver, + cscf->resolver_timeout) + != NGX_OK) + { + return NGX_ERROR; + } + } + } h = ngx_array_push(&cmcf->phases[NGX_STREAM_SSL_PHASE].handlers); if (h == NULL) { diff --git a/src/stream/ngx_stream_ssl_module.h b/src/stream/ngx_stream_ssl_module.h index 6f6d9aec3e9..ef314271610 100644 --- a/src/stream/ngx_stream_ssl_module.h +++ b/src/stream/ngx_stream_ssl_module.h @@ -53,6 +53,10 @@ typedef struct { ngx_flag_t session_tickets; ngx_array_t *session_ticket_keys; + + ngx_uint_t ocsp; + ngx_str_t ocsp_responder; + ngx_shm_zone_t *ocsp_cache_zone; } ngx_stream_ssl_srv_conf_t; From fb89d50eeb19d42d83144ff76c80d20e80c41aca Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Thu, 22 Aug 2024 14:57:46 +0400 Subject: [PATCH 153/279] Stream: OCSP stapling. --- src/stream/ngx_stream_ssl_module.c | 78 ++++++++++++++++++++++++++---- src/stream/ngx_stream_ssl_module.h | 5 ++ 2 files changed, 73 insertions(+), 10 deletions(-) diff --git a/src/stream/ngx_stream_ssl_module.c b/src/stream/ngx_stream_ssl_module.c index 37f9c547d6c..072e74917cf 100644 --- a/src/stream/ngx_stream_ssl_module.c +++ b/src/stream/ngx_stream_ssl_module.c @@ -243,6 +243,34 @@ static ngx_command_t ngx_stream_ssl_commands[] = { 0, NULL }, + { ngx_string("ssl_stapling"), + NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_FLAG, + ngx_conf_set_flag_slot, + NGX_STREAM_SRV_CONF_OFFSET, + offsetof(ngx_stream_ssl_srv_conf_t, stapling), + NULL }, + + { ngx_string("ssl_stapling_file"), + NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_TAKE1, + ngx_conf_set_str_slot, + NGX_STREAM_SRV_CONF_OFFSET, + offsetof(ngx_stream_ssl_srv_conf_t, stapling_file), + NULL }, + + { ngx_string("ssl_stapling_responder"), + NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_TAKE1, + ngx_conf_set_str_slot, + NGX_STREAM_SRV_CONF_OFFSET, + offsetof(ngx_stream_ssl_srv_conf_t, stapling_responder), + NULL }, + + { ngx_string("ssl_stapling_verify"), + NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_FLAG, + ngx_conf_set_flag_slot, + NGX_STREAM_SRV_CONF_OFFSET, + offsetof(ngx_stream_ssl_srv_conf_t, stapling_verify), + NULL }, + { ngx_string("ssl_conf_command"), NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_TAKE2, ngx_conf_set_keyval_slot, @@ -809,6 +837,8 @@ ngx_stream_ssl_create_srv_conf(ngx_conf_t *cf) * sscf->ciphers = { 0, NULL }; * sscf->shm_zone = NULL; * sscf->ocsp_responder = { 0, NULL }; + * sscf->stapling_file = { 0, NULL }; + * sscf->stapling_responder = { 0, NULL }; */ sscf->handshake_timeout = NGX_CONF_UNSET_MSEC; @@ -826,6 +856,8 @@ ngx_stream_ssl_create_srv_conf(ngx_conf_t *cf) sscf->session_ticket_keys = NGX_CONF_UNSET_PTR; sscf->ocsp = NGX_CONF_UNSET_UINT; sscf->ocsp_cache_zone = NGX_CONF_UNSET_PTR; + sscf->stapling = NGX_CONF_UNSET; + sscf->stapling_verify = NGX_CONF_UNSET; return sscf; } @@ -885,6 +917,12 @@ ngx_stream_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) ngx_conf_merge_ptr_value(conf->ocsp_cache_zone, prev->ocsp_cache_zone, NULL); + ngx_conf_merge_value(conf->stapling, prev->stapling, 0); + ngx_conf_merge_value(conf->stapling_verify, prev->stapling_verify, 0); + ngx_conf_merge_str_value(conf->stapling_file, prev->stapling_file, ""); + ngx_conf_merge_str_value(conf->stapling_responder, + prev->stapling_responder, ""); + conf->ssl.log = cf->log; if (conf->certificates) { @@ -983,18 +1021,18 @@ ngx_stream_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) { return NGX_CONF_ERROR; } + } - if (ngx_ssl_trusted_certificate(cf, &conf->ssl, - &conf->trusted_certificate, - conf->verify_depth) - != NGX_OK) - { - return NGX_CONF_ERROR; - } + if (ngx_ssl_trusted_certificate(cf, &conf->ssl, + &conf->trusted_certificate, + conf->verify_depth) + != NGX_OK) + { + return NGX_CONF_ERROR; + } - if (ngx_ssl_crl(cf, &conf->ssl, &conf->crl) != NGX_OK) { - return NGX_CONF_ERROR; - } + if (ngx_ssl_crl(cf, &conf->ssl, &conf->crl) != NGX_OK) { + return NGX_CONF_ERROR; } if (conf->ocsp) { @@ -1055,6 +1093,17 @@ ngx_stream_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) return NGX_CONF_ERROR; } + if (conf->stapling) { + + if (ngx_ssl_stapling(cf, &conf->ssl, &conf->stapling_file, + &conf->stapling_responder, conf->stapling_verify) + != NGX_OK) + { + return NGX_CONF_ERROR; + } + + } + if (ngx_ssl_conf_commands(cf, &conf->ssl, conf->conf_commands) != NGX_OK) { return NGX_CONF_ERROR; } @@ -1454,6 +1503,15 @@ ngx_stream_ssl_init(ngx_conf_t *cf) cscf = cscfp[s]->ctx->srv_conf[ngx_stream_core_module.ctx_index]; + if (sscf->stapling) { + if (ngx_ssl_stapling_resolver(cf, &sscf->ssl, cscf->resolver, + cscf->resolver_timeout) + != NGX_OK) + { + return NGX_ERROR; + } + } + if (sscf->ocsp) { if (ngx_ssl_ocsp_resolver(cf, &sscf->ssl, cscf->resolver, cscf->resolver_timeout) diff --git a/src/stream/ngx_stream_ssl_module.h b/src/stream/ngx_stream_ssl_module.h index ef314271610..e6769426c8e 100644 --- a/src/stream/ngx_stream_ssl_module.h +++ b/src/stream/ngx_stream_ssl_module.h @@ -57,6 +57,11 @@ typedef struct { ngx_uint_t ocsp; ngx_str_t ocsp_responder; ngx_shm_zone_t *ocsp_cache_zone; + + ngx_flag_t stapling; + ngx_flag_t stapling_verify; + ngx_str_t stapling_file; + ngx_str_t stapling_responder; } ngx_stream_ssl_srv_conf_t; From 900f4dc48c07d6f6961744fd152d3afb10a2bd9f Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Thu, 29 Aug 2024 16:03:27 +0400 Subject: [PATCH 154/279] Removed .hgtags file. --- .hgtags | 482 -------------------------------------------------------- 1 file changed, 482 deletions(-) delete mode 100644 .hgtags diff --git a/.hgtags b/.hgtags deleted file mode 100644 index 2c9cb3bc6bc..00000000000 --- a/.hgtags +++ /dev/null @@ -1,482 +0,0 @@ -551102312e19b704cd22bd7254a9444b9ea14e96 release-0.1.0 -23fb87bddda14ce9faec90f774085634106aded4 release-0.1.1 -295d97d70c698585705345f1a8f92b02e63d6d0d release-0.1.2 -ded1284520cc939ad5ae6ddab39925375e64237d release-0.1.3 -0491b909ef7612d8411f1f59054186c1f3471b52 release-0.1.4 -a88a3e4e158fade0aaa6f3eb25597d5ced2c1075 release-0.1.5 -1f31dc6d33a3a4e65240b08066bf186df9e33b79 release-0.1.6 -5aecc125bc33d81d6214c91d73eb44230a903dde release-0.1.7 -bbd6b0b4a2b15ef8c8f1aaf7b027b6da47303524 release-0.1.8 -2ff194b74f1e60cd04670986973e3b1a6aa3bece release-0.1.9 -31ee1b50354fb829564b81a6f34e8d6ceb2d3f48 release-0.1.10 -8e8f3af115b5b903b2b8f3335de971f18891246f release-0.1.11 -c3c2848fc081e19aec5ffa97e468ad20ddb81df0 release-0.1.12 -ad1e9ebf93bb5ae4c748d471fad2de8a0afc4d2a release-0.1.13 -c5240858380136a67bec261c59b1532560b57885 release-0.1.14 -fd661d14a7fad212e326a7dad6234ea0de992fbf release-0.1.15 -621229427cba1b0af417ff2a101fc4f17a7d93c8 release-0.1.16 -4ebe09b07e3021f1a63b459903ec58f162183b26 release-0.1.17 -31ff3e943e1675a2caf745ba7a981244445d4c98 release-0.1.18 -45a460f82aec80b0f61136aa09f412436d42203a release-0.1.19 -0f836f0288eee4980f57736d50a7a60fa082d8e9 release-0.1.20 -975f62e77f0244f1b631f740be77c72c8f2da1de release-0.1.21 -fc9909c369b2b4716304ac8e38da57b8fb781211 release-0.1.22 -d7c90bb5ce83dab08715e98f9c7b81c7df4b37be release-0.1.23 -64d9afb209da0cd4a917202b7b77e51cc23e2229 release-0.1.24 -d4ea69372b946dc4ec37fc3f5ddd93ff7c3da675 release-0.1.25 -b1648294f6935e993e436fd8a68bca75c74c826d release-0.1.26 -ee66921ecd47a7fa459f70f4a9d660f91f6a1b94 release-0.1.27 -cd3117ad9aab9c58c6f7e677e551e1adbdeaba54 release-0.1.28 -9b8c906f6e63ec2c71cecebfff35819a7d32227d release-0.1.29 -c12967aadd8726daf2d85e3f3e622d89c42db176 release-0.1.30 -fbbf16224844e7d560c00043e8ade8a560415bba release-0.1.31 -417a087c9c4d9abb9b0b9b3f787aff515c43c035 release-0.1.32 -dadfa78d227027348d7f9d1e7b7093d06ba545a0 release-0.1.33 -12234c998d83bfbbaa305273b3dd1b855ca325dc release-0.1.34 -6f00349b98e5f706b82115c6e4dc84456fc0d770 release-0.1.35 -2019117e6b38cc3e89fe4f56a23b271479c627a6 release-0.1.36 -09b42134ac0c42625340f16628e29690a04f8db5 release-0.1.37 -7fa11e5c6e9612ecff5eb58274cc846ae742d1d2 release-0.1.38 -e5d7d0334fdb946133c17523c198800142ac9fe9 release-0.1.39 -c3bd8cdabb8f73e5600a91f198eb7df6fac65e92 release-0.1.40 -d6e48c08d718bf5a9e58c20a37e8ae172bff1139 release-0.1.41 -563ad09abf5042eb41e8ecaf5b4e6c9deaa42731 release-0.1.42 -c9ad0d9c7d59b2fa2a5fe669f1e88debd03e6c04 release-0.1.43 -371c1cee100d7a1b0e6cad4d188e05c98a641ee7 release-0.1.44 -b09ee85d0ac823e36861491eedfc4dfafe282997 release-0.1.45 -511a89da35ada16ae806667d699f9610b4f8499a release-0.2.0 -0148586012ab3dde69b394ec5a389d44bb11c869 release-0.2.1 -818fbd4750b99d14d2736212c939855a11b1f1ef release-0.2.2 -e16a8d574da511622b97d6237d005f40f2cddb30 release-0.2.3 -483cca23060331f2078b1c2984870d80f288ad41 release-0.2.4 -45033d85b30e3f12c407b7cfc518d76e0eda0263 release-0.2.5 -7bd37aef1e7e87858c12b124e253e98558889b50 release-0.2.6 -ecd9c160f25b7a7075dd93383d98a0fc8d8c0a41 release-0.3.0 -c1f965ef97188fd7ef81342dcf8719da18c554d2 release-0.3.1 -e48ebafc69393fc94fecfdf9997c4179fd1ce473 release-0.3.2 -9c2f3ed7a24711d3b42b124d5f831155c8beff95 release-0.3.3 -7c1369d37c7eb0017c28ebcaa0778046f5aafdcc release-0.3.4 -1af2fcb3be8a63796b6b23a488049c92a6bc12f4 release-0.3.5 -174f1e853e1e831b01000aeccfd06a9c8d4d95a2 release-0.3.6 -458b6c3fea65a894c99dd429334a77bb164c7e83 release-0.3.7 -58475592100cb792c125101b6d2d898f5adada30 release-0.3.8 -fcd6fc7ff7f9b132c35193d834e6e7d05026c716 release-0.3.9 -4d9ea73a627a914d364e83e20c58eb1283f4031d release-0.3.10 -4c5c2c55975c1152b5ca5d5d55b32d4dd7945f7a release-0.3.11 -326634fb9d47912ad94221dc2f8fa4bec424d40c release-0.3.12 -4e296b7d25bf62390ca2afb599e395426b94f785 release-0.3.13 -401de5a43ba5a8acdb9c52465193c0ea7354afe7 release-0.3.14 -284cc140593bb16ac71094acd509ab415ff4837d release-0.3.15 -d4e858a5751a7fd08e64586795ed7d336011fbc0 release-0.3.16 -8c0cdd81580eb76d774cfc5724de68e7e5cbbdc2 release-0.3.17 -425af804d968f30eeff01e33b808bc2e8c467f2c release-0.3.18 -ebc68d8ca4962fe3531b7e13444f7ac4395d9c6e release-0.3.19 -9262f520ce214d3d5fd7c842891519336ef85ca6 release-0.3.20 -869b6444d2341a587183859d4df736c7f3381169 release-0.3.21 -77f77f53214a0e3a68fef8226c15532b54f2c365 release-0.3.22 -858700ae46b453ea111b966b6d03f2c21ddcb94e release-0.3.23 -5dac8c7fb71b86aafed8ea352305e7f85759f72e release-0.3.24 -77cdfe394a94a625955e7585e09983b3af9b889b release-0.3.25 -608cf78b24ef7baaf9705e4715a361f26bb16ba9 release-0.3.26 -3f8a2132b93d66ac19bec006205a304a68524a0b release-0.3.27 -c73c5c58c619c22dd3a5a26c91bb0567a62c6930 release-0.3.28 -5ef026a2ac7481f04154f29ab49377bf99aaf96f release-0.3.29 -51b27717f140b71a2e9158807d79da17c888ce4c release-0.3.30 -7a16e281c01f1c7ab3b79c64b43ddb754ea7935e release-0.3.31 -93e85a79757c49d502e42a1cb8264a0f133b0b00 release-0.3.32 -0216fd1471f386168545f772836156761eddec08 release-0.3.33 -fbed40ce7cb4fd7203fecc22a617b9ce5b950fb3 release-0.3.34 -387450de0b4d21652f0b6242a5e26a31e3be8d8c release-0.3.35 -65bf042c0b4f39f18a235464c52f980e9fa24f6b release-0.3.36 -5d2b8078c1c2593b95ec50acfeeafbefa65be344 release-0.3.37 -f971949ffb585d400e0f15508a56232a0f897c80 release-0.3.38 -18268abd340cb351e0c01b9c44e9f8cc05492364 release-0.3.39 -e60fe4cf1d4ea3c34be8c49047c712c6d46c1727 release-0.3.40 -715d243270806d38be776fc3ed826d97514a73d6 release-0.3.41 -5e8fb59c18c19347a5607fb5af075fe1e2925b9a release-0.3.42 -947c6fd27699e0199249ad592151f844c8a900b0 release-0.3.43 -4946078f0a79e6cc952d3e410813aac9b8bda650 release-0.3.44 -95d7da23ea5315a6e9255ce036ed2c51f091f180 release-0.3.45 -1e720b0be7ecd92358da8a60944669fa493e78cd release-0.3.46 -39b7d7b33c918d8f4abc86c4075052d8c19da3c7 release-0.3.47 -7cbef16c71a1f43a07f8141f02e0135c775f0f5b release-0.3.48 -4c8cd5ae5cc100add5c08c252d991b82b1838c6b release-0.3.49 -400711951595aef7cd2ef865b84b31df52b15782 release-0.3.50 -649c9063d0fda23620eaeaf0f6393be0a672ebe7 release-0.3.51 -9079ee4735aefa98165bb2cb26dee4f58d58c1d7 release-0.3.52 -6d5c1535bb9dcd891c5963971f767421a334a728 release-0.3.53 -5fd7a5e990477189c40718c8c3e01002a2c20b81 release-0.3.54 -63a820b0bc6ca629c8e45a069b52d622ddc27a2d release-0.3.55 -562806624c4afb1687cba83bc1852f5d0fecbac3 release-0.3.56 -cec32b3753acf610ac1a6227d14032c1a89d6319 release-0.3.57 -b80f94fa2197b99db5e033fec92e0426d1fe5026 release-0.3.58 -e924670896abe2769ea0fcfd2058b405bed8e8ec release-0.3.59 -921a7ce4baf42fd1091b7e40f89c858c6b23053e release-0.3.60 -df95dcff753a6dc5e94257302aea02c18c7a7c87 release-0.3.61 -7e24168b0853ee7e46c9c7b943ef077dc64f17f5 release-0.4.0 -8183d4ba50f8500465efb27e66dd23f98775dd21 release-0.4.1 -610267a772c7bf911b499d37f66c21ce8f2ebaf7 release-0.4.2 -39dd0b045441e21512e0a6061a03d0df63414d8b release-0.4.3 -5e42c1615f4de0079bd4d8913886d588ce6a295d release-0.4.4 -40266f92b829a870808b3d4ee54c8fccdecbd2d6 release-0.4.5 -56e33c6efee7ff63cdc52bd1cf172bde195079df release-0.4.6 -119bad43bfd493400c57a05848eada2c35a46810 release-0.4.7 -0f404f82a1343cb4e4b277a44e3417385798e5e5 release-0.4.8 -d24a717314365c857b9f283d6072c2a427d5e342 release-0.4.9 -d6f0a00015fdef861fd67fb583b9690638650656 release-0.4.10 -e372368dadd7b2ecd0182b2f1b11db86fc27b2c3 release-0.4.11 -fd57967d850d2361072c72562d1ed03598473478 release-0.4.12 -979045fdcbd20cf7188545c1c589ff240251f890 release-0.4.13 -93c94cfa9f78f0a5740595dde4466ec4fba664f8 release-0.4.14 -589ee12e8d7c2ae5e4f4676bcc7a1279a76f9e8e release-0.5.0 -13416db8a807e5acb4021bc3c581203de57e2f50 release-0.5.1 -06c58edc88831fb31c492a8eddcf2c6056567f18 release-0.5.2 -e2ac5fa41bcba14adbbb722d45c083c30c07bb5c release-0.5.3 -393dbc659df15ccd411680b5c1ce87ed86d4c144 release-0.5.4 -38cc7bd8e04f2c519fd4526c12841a876be353cb release-0.5.5 -6d1fcec2ea79101c756316c015f72e75f601a5ab release-0.5.6 -aed8a9de62456c4b360358bc112ccca32ce02e8d release-0.5.7 -7642f45af67d805452df2667486201c36efaff85 release-0.5.8 -779216610662c3a459935d506f66a9b16b9c9576 release-0.5.9 -9eeb585454f3daa30cf768e95c088a092fe229b9 release-0.5.10 -bb491c8197e38ca10ae63b1f1ecb36bf6fdaf950 release-0.5.11 -613369e08810f36bbcc9734ef1059a03ccbf5e16 release-0.5.12 -bd796ef5c9c9dd34bfac20261b98685e0410122a release-0.5.13 -8a730c49f906d783b47e4b44d735efd083936c64 release-0.5.14 -cb447039152d85e9145139ff2575a6199b9af9d4 release-0.5.15 -64854c7c95d04f838585ca08492823000503fa61 release-0.5.16 -d1ffcf84ea1244f659145c36ff28de6fcdf528b2 release-0.5.17 -796a6e30ca9d29504195c10210dbc8deced0ae83 release-0.5.18 -1f81c711d2a039e1f93b9b515065a2235372d455 release-0.5.19 -8e8f6082654aedb4438c8fca408cfc316c7c5a2a release-0.5.20 -e9551132f7dd40da5719dd5bcf924c86f1436f85 release-0.5.21 -533a252896c4d1cff1586ae42129d610f7497811 release-0.5.22 -f461a49b6c747e0b67f721f2be172902afea5528 release-0.5.23 -2d5ef73671f690b65bf6d9e22e7155f68f484d5a release-0.5.24 -77bf42576050862c268e267ef3e508b145845a25 release-0.5.25 -2aefee4d4ed69eb7567680bf27a2efd212232488 release-0.6.0 -7ac0fe9bec9a2b5f8e191f6fdd6922bfd916a6cb release-0.6.1 -4882735ebc71eeec0fbfe645bdfdb31306872d82 release-0.6.2 -b94731c73d0922f472ff938b9d252ba29020f20c release-0.6.3 -13e649b813d6ccba5db33a61e08ebe09d683cd5b release-0.6.4 -80de622646b0059fd4c553eff47c391bf7503b89 release-0.6.5 -3b05edb2619d5935023b979ee7a9611b61b6c9e5 release-0.6.6 -1dcfd375100c4479611f71efb99271d0a3059215 release-0.6.7 -0228185d4c5772947b842e856ad74cf7f7fd52f3 release-0.6.8 -d1879c52326ecac45c713203670f54220879911e release-0.6.9 -5a80c6ccbe2ad24fa3d4ff6f9fe4a2b07408d19d release-0.6.10 -f88a8b0b39601b19cd740e4db614ab0b5b874686 release-0.6.11 -5557460a7247a1602ae96efd1d0ccf781344cb58 release-0.6.12 -451b02cc770a794cd41363461b446948ae1d8bc8 release-0.6.13 -537b6ef014c4a133e0ab0b7dc817508e0647e315 release-0.6.14 -5e68764f0d6e91a983170fa806e7450a9e9b33fe release-0.6.15 -158aa4e8cc46fcf9504a61469d22daf3476b17bf release-0.6.16 -d8fcca555542619228d9fab89e1665b993f8c3ee release-0.6.17 -60707ebc037086cf004736a0d4979e2a608da033 release-0.6.18 -3c2a99d3a71af846855be35e62edb9a12f363f44 release-0.6.19 -3e0a27f9358ffc1b5249e0ea2311ce7da5c8967e release-0.6.20 -143f4d65b1c875d6563ccb7f653d9157afc72194 release-0.6.21 -95e6160d2b7d0af8ffd1b95a23cadadf8f0b3f6d release-0.6.22 -69a03d5e3b6e6660079ef1ef172db7ac08d8370e release-0.6.23 -3e2a58fb48f1e1a99ebf851e0d47a7034c52ae22 release-0.6.24 -3b8607c05a8bebcfa59235c2126a70d737f0ccf5 release-0.6.25 -07ad5b2606614c4be4ee720c46cf4af126059d31 release-0.6.26 -be531addfabe5214f409d457140c1038af10d199 release-0.6.27 -58f05255d3a345d04baef5cff0ca1ae0ac7ecebb release-0.6.28 -eb2bd21dc8d03f6c94016f04ffb9adaf83a2b606 release-0.6.29 -55408deb3cd171efa9b81d23d7a1dd1ccde0b839 release-0.6.30 -d4288915bba73c4c3c9cf5d39d34e86879eb2b45 release-0.6.31 -0a189588830b8629c4dfea68feb49af36b59e4a9 release-0.7.0 -6ab27a06f3346cf9ec8737f5dbcc82dd4031e30f release-0.7.1 -a07e258cef3b0a0b6e76a6ff4ba4651c5facc85a release-0.7.2 -9992c4583513d2804fc2e7fec860fbc7ab043009 release-0.7.3 -4dc24d50230fbadfc037a414a86390db2de69dd2 release-0.7.4 -9527137b4354a648a229c7169850c7c65272c00d release-0.7.5 -c2f0f7cf306f302254beae512bda18713922375c release-0.7.6 -bbcf6d75556fdcee8bd4aba8f6c27014be9920ee release-0.7.7 -43bde71f0bbe5a33b161760d7f9f980d50386597 release-0.7.8 -769f0dd7081e9011394f264aa22aa66fd79730d8 release-0.7.9 -511edfa732da637f5f0c9476335df7dca994706d release-0.7.10 -0e7023bf6b2461309c29885935443449a41be807 release-0.7.11 -9ad1bd2b21d93902863807528e426862aedee737 release-0.7.12 -d90ea21e24ea35379aef50c5d70564158e110a15 release-0.7.13 -c07d2d20d95c83d804079bbdcecbce4a0c8282f0 release-0.7.14 -0cd7bb051f67eac2b179fb9f9cc988b9ba18ed76 release-0.7.15 -eab2e87deba73ae6abd9cc740e8d4365bed96322 release-0.7.16 -91d7a9eb8ade90e9421d7b1e3c2e47a6bc427876 release-0.7.17 -fc10f7b5cb1305fb930f8ac40b46882d0828d61e release-0.7.18 -9dba9779e37e5969a2d408c792084fd7acfec062 release-0.7.19 -61838d1bcbddc7bc4dd9f30d535573a6fddca8f9 release-0.7.20 -5f665d0fa6a5f6e748157f2ccbc445b2db8125d0 release-0.7.21 -24763afa5efe91e54f00b2ae5b87666eb6c08c3b release-0.7.22 -0562fb355a25266150cbe8c8d4e00f55e3654df3 release-0.7.23 -19c452ecd083550816873a8a31eb3ed9879085e6 release-0.7.24 -46b68faf271d6fdcaaf3ad2c69f6167ea9e9fa28 release-0.7.25 -d04bfca0c7e3ae2e4422bc1d383553139d6f0a19 release-0.7.26 -9425d9c7f8ead95b00a3929a9a5e487e0e3c8499 release-0.7.27 -fbc3e7e8b3ee756568a875f87d8a954a2f9d3bf6 release-0.7.28 -5176dfdf153fc785b18604197d58806f919829ad release-0.7.29 -87e07ccdf0a4ec53458d9d7a4ea66e1239910968 release-0.7.30 -9fddd7e1a7a27f8463867f41a461aad57df461b2 release-0.7.31 -780b2ba1ec6daf6e3773774e26b05b9ff0d5483e release-0.7.32 -83027471a25385b1c671968be761e9aa7a8591a7 release-0.7.33 -1e9a362c3dcee221ca6e34308c483ed93867aca2 release-0.7.34 -c7ee9e15717b54ead5f4a554686e74abe66c6b07 release-0.7.35 -b84548abe9b9d4f4e203f848696e52c8c82c308f release-0.7.36 -3286f0bab8e77dbc7ebb370b1dc379592ccff123 release-0.7.37 -11a4e2ed5b166b9c9f119171aa399a9e3aa4684a release-0.7.38 -f822655d4120629977794c32d3b969343b6c30db release-0.7.39 -8a350e49d2b6751296db6d8e27277ccf63ed412a release-0.7.40 -c4a56c197eeafd71fc1caef7a9d890a330e3c23d release-0.7.41 -a9575a57a5443df39611774cf3840e9088132b0e release-0.7.42 -7503d95d6eadad14c28b2db183ba09848265274b release-0.7.43 -9be652e9114435fc6f1fdec84c0458d56702db91 release-0.7.44 -797e070d480a34b31ddac0d364784773f1bbbcf9 release-0.7.45 -9b5037e7ec7db25875c40f9d1cf20a853388b124 release-0.7.46 -d1d0e6d7ff0ca3c0dd1be1ef1cfff2e3fd0b4e1c release-0.7.47 -9816fb28eda599bfd53940e6d3b6617d1ecb6323 release-0.7.48 -452b9d09df8e3f2fb04b2a33d04d2f3a6436eb34 release-0.7.49 -e4350efa7cf7a0e868c2236a1137de8a33bd8ec6 release-0.7.50 -f51f2bec766c8b6d7e1799d904f18f8ea631bd44 release-0.7.51 -18e39e566781c9c187e2eb62bebd9d669d68f08c release-0.7.52 -b073eaa1dcea296a3488b83d455fab6621a73932 release-0.7.53 -01c6fe6c2a55998434cd3b05dd10ca487ac3fb6c release-0.7.54 -3ed9377e686f2521e6ec15873084381033fb490d release-0.7.55 -a1e44954549c35023b409f728c678be8bf898148 release-0.7.56 -fbb1918a85e38a7becdb1a001dbaf5933f23a919 release-0.7.57 -87f4a49a9cc34a5b11c8784cc5ea89e97b4b2bd8 release-0.7.58 -0c22cb4862c8beb4ee1b9e4627125162a29a5304 release-0.7.59 -82d56c2425ef857cd430b8530a3f9e1127145a67 release-0.8.0 -f4acb784b53cd952559567971b97dde1e818a2b6 release-0.8.1 -b3503597c1a0f0f378afdc5e5e5b85e2c095a4be release-0.8.2 -c98da980514a02ba81c421b25bf91803ffffddf3 release-0.8.3 -db34ec0c53c4b9dec12ffdf70caf89a325ab9577 release-0.8.4 -0914802433b8678ba2cdf91280766f00f4b9b76e release-0.8.5 -ff52ee9e6422f3759f43a442b7ba615595b3a3d4 release-0.8.6 -7607237b4829fff1f60999f4663c50ed9d5182f7 release-0.8.7 -1cef1807bc12cb05ac52fb0e7a0f111d3760b569 release-0.8.8 -a40f8475511d74a468ade29c1505e8986600d7a3 release-0.8.9 -2d9faf2260df6c3e5d4aa1781493c31f27a557d0 release-0.8.10 -d0d61c32331a6505381b5218318f7b69db167ca8 release-0.8.11 -ca7a1c6c798a7eb5b294d4ac3179ec87ecf297d3 release-0.8.12 -81c8277cd8ed55febcb2dd9d9213076f6c0ccb09 release-0.8.13 -3089486a8dc5844b5b6e9f78d536b4b26f7ffa16 release-0.8.14 -d364c2c12dd9723a2dfac3f096f5e55d4cfe6838 release-0.8.15 -52163a1027c3efd6b4c461b60a2ca6266c23e193 release-0.8.16 -06564e9a2d9ec5852132c212e85eda0bf1300307 release-0.8.17 -7aaa959da85e09e29bcac3b1cadec35b0a25b64d release-0.8.18 -4bc73c644329a510da4e96b7241b80ead7772f83 release-0.8.19 -ea3d168fb99c32a5c3545717ecc61e85a375e5dd release-0.8.20 -27951ca037e63dae45ff5b6279124c224ae1255a release-0.8.21 -d56c8b5df517c2bf6e7bc2827b8bf3e08cda90e1 release-0.8.22 -3c6ac062b379b126212cbb27e98a3c8275ef381a release-0.8.23 -89b9173476de14688b1418fbf7df10f91d1719ef release-0.8.24 -aa550cb4159ae0d566006e091fb1c7a888771050 release-0.8.25 -06ce92293f6a65651b08c466f90f55bd69984b98 release-0.8.26 -ea50b0d79ef1d7d901cd0e4dcd7373447849d719 release-0.8.27 -e68b1c35cad86105ff1c5b240f53442f4c36356e release-0.8.28 -78d3582a30afe63fc0adb17c3ac8891a64e47146 release-0.8.29 -9852c5965a3292a1b6127dbb4da9fce4912d898a release-0.8.30 -4f84115914490e572bcbee5069157b7334df2744 release-0.8.31 -59dee6f7f3afeb1fad6ed5983756e48c81ad2a5c release-0.8.32 -a4456378d234c07038456cf32bfe3c651f1d5e82 release-0.8.33 -21cb50799a20575a42f9733342d37a426f79db4d release-0.8.34 -7cb3cb8d78ef7ae63561733ed91fd07933896bc8 release-0.8.35 -aed68639d4eb6afe944b7fb50499c16f7f3f503c release-0.8.36 -265b7fd2ae21c75bbffa5115b83a0123d6c4acb4 release-0.8.37 -fa5f1ca353c0c5aa5415f51d72fd7bbcc02d1ed7 release-0.8.38 -af10bf9d4c6532850aa1f70cdf7504bd109b284c release-0.8.39 -4846ec9f83cb5bc4c8519d5641b35fb9b190430c release-0.8.40 -718b4cb3faf7efe4e0648140f064bf7a92c3f7e8 release-0.8.41 -b5a3065749093282ddd19845e0b77ffc2e54333e release-0.8.42 -34df9fb22fed415cdad52def04095dc6d4b48222 release-0.8.43 -00ec8cd76fb89af27363b76c40d9f88bf4679c3b release-0.8.44 -e16dd52a0d226c23dcae9a11252564a04753bbed release-0.8.45 -f034d9173df0a433e0bbcf5974f12ea9eb9076c0 release-0.8.46 -4434dc967087315efcd0658206a67fe6c85528f3 release-0.8.47 -0b65c962e0cd6783a854877b52c903cb058eec8c release-0.8.48 -a2b7e94b9807e981866bf07e37b715847d1b7120 release-0.8.49 -e7bdb8edc1bab2bc352a9fb6ce765c46575c35bf release-0.8.50 -21dacebd12f65cb57ceb8d2688db5b07fad6e06d release-0.8.51 -67dd7533b99c8945b5b8b5b393504d4e003a1c50 release-0.8.52 -010468d890dbac33a4cae6dfb2017db70721b2fe release-0.8.53 -62b599022a2fa625b526c2ad1711dc6db7d66786 release-0.9.0 -71281dd73b17a0ead5535d531afaee098da723cb release-0.9.1 -16cff36b0e49fc9fdeee13b2e92690286bcc1b3d release-0.9.2 -b7b306325972661117694879d3e22faf4cf0df32 release-0.9.3 -fe671505a8ea86a76f0358b3ec4de84a9037ac2b release-0.9.4 -70542931bc5436d1bbd38f152245d93ac063968d release-0.9.5 -27e2f3b7a3db1819c5d0ba28327ceaba84a13c4e release-0.9.6 -657d05d63915ce2f6c4d763091059f5f85bb10e5 release-0.9.7 -e0fd9f36005923b8f98d1ba1ea583cb7625f318f release-1.0.0 -f8f89eb4e0c27e857ec517d893d4f9a454985084 release-1.0.1 -c50df367648e53d55e80b60a447c9c66caa0d326 release-1.0.2 -80d586db316512b5a9d39f00fe185f7f91523f52 release-1.0.3 -c9c2805ac9245cc48ce6efeba2b4a444f859d6aa release-1.0.4 -fa2c37b1122c2c983b6e91d1188e387d72dde4d6 release-1.0.5 -f31aea5b06654c9163be5acd6d9b7aaf0fdf6b33 release-1.1.0 -44bf95f670656fae01ccb266b3863843ea13d324 release-1.1.1 -da1289482a143dfa016769649bdff636c26f53c8 release-1.1.2 -bac8ba08a6570bac2ecd3bf2ad64b0ac3030c903 release-1.1.3 -911060bc8221d4113a693ae97952a1fa88663ca8 release-1.1.4 -e47531dfabbf8e5f8b8aff9ff353642ea4aa7abb release-1.1.5 -f9ddecfe331462f870a95e4c1c3ba1bb8f19f2d3 release-1.1.6 -378c297bb7459fb99aa9c77decac0d35391a3932 release-1.1.7 -71600ce67510af093d4bc0117a78b3b4678c6b3a release-1.1.8 -482d7d907f1ab92b78084d8b8631ed0eb7dd08f7 release-1.1.9 -c7e65deabf0db5109e8d8f6cf64cd3fb7633a3d1 release-1.1.10 -9590f0cf5aab8e6e0b0c8ae59c70187b2b97d886 release-1.1.11 -ade8fc136430cfc04a8d0885c757968b0987d56c release-1.1.12 -6a6836e65827fd3cb10a406e7bbbe36e0dad8736 release-1.1.13 -6845f4ac909233f5a08ed8a51de137713a888328 release-1.1.14 -2397e9c72f1bc5eac67006e12ad3e33e0ea9ba74 release-1.1.15 -7b7c49639a7bceecabf4963c60b26b65a77d6ce0 release-1.1.16 -f7e1113a9a1648cad122543e7080e895cf2d88f4 release-1.1.17 -2b22743c3079b41233ded0fc35af8aa89bcfab91 release-1.1.18 -0f0b425659e0b26f5bc8ea14a42dbf34de2eaba6 release-1.1.19 -f582d662cc408eb7a132c21f4b298b71d0701abb release-1.2.0 -9ee68d629722f583d43d92271f2eb84281afc630 release-1.3.0 -61b6a3438afef630774e568eefd89c53e3b93287 release-1.3.1 -7ccd50a0a455f2f2d3b241f376e1193ad956196d release-1.2.1 -0000000000000000000000000000000000000000 release-1.2.1 -50107e2d96bbfc2c59e46f889b1a5f68dd10cf19 release-1.3.2 -2c5e1e88c8cf710caf551c5c67eba00443601efe release-1.3.3 -a43447fb82aa03eabcd85352758ae14606a84d35 release-1.3.4 -90f3b4ea7992a7bf9385851a3e77173363091eea release-1.3.5 -3aeb14f88daeb973e4708310daa3dc68ac1200f7 release-1.3.6 -dafd375f1c882b15fa4a9b7aa7c801c55082395e release-1.3.7 -ab7ce0eb4cf78a656750ab1d8e55ef61f7e535ec release-1.3.8 -1b1a9337a7399ad3cdc5e3a2f9fbaaec990271d5 release-1.3.9 -2c053b2572694eb9cd4aed26a498b6cb1f51bbcc release-1.3.10 -36409ac209872ce53019f084e4e07467c5d9d25e release-1.3.11 -560dc55e90c13860a79d8f3e0d67a81c7b0257bb release-1.3.12 -dc195ffe0965b2b9072f8e213fe74ecce38f6773 release-1.3.13 -e04428778567dd4de329bbbe97ad653e22801612 release-1.3.14 -cd84e467c72967b9f5fb4d96bfc708c93edeb634 release-1.3.15 -23159600bdea695db8f9d2890aaf73424303e49c release-1.3.16 -7809529022b83157067e7d1e2fb65d57db5f4d99 release-1.4.0 -48a84bc3ff074a65a63e353b9796ff2b14239699 release-1.5.0 -99eed1a88fc33f32d66e2ec913874dfef3e12fcc release-1.5.1 -5bdca4812974011731e5719a6c398b54f14a6d61 release-1.5.2 -644a079526295aca11c52c46cb81e3754e6ad4ad release-1.5.3 -376a5e7694004048a9d073e4feb81bb54ee3ba91 release-1.5.4 -60e0409b9ec7ee194c6d8102f0656598cc4a6cfe release-1.5.5 -70c5cd3a61cb476c2afb3a61826e59c7cda0b7a7 release-1.5.6 -9ba2542d75bf62a3972278c63561fc2ef5ec573a release-1.5.7 -eaa76f24975948b0ce8be01838d949122d44ed67 release-1.5.8 -5a1759f33b7fa6270e1617c08d7e655b7b127f26 release-1.5.9 -b798fc020e3a84ef68e6c9f47865a319c826d33c release-1.5.10 -f995a10d4c7e9a817157a6ce7b753297ad32897e release-1.5.11 -97b47d95e4449cbde976657cf8cbbc118351ffe0 release-1.5.12 -fd722b890eabc600394349730a093f50dac31639 release-1.5.13 -d161d68df8be32e5cbf72b07db1a707714827803 release-1.7.0 -0351a6d89c3dbcc7a76295024ba6b70e27b9a497 release-1.7.1 -0bd223a546192fdf2e862f33938f4ec2a3b5b283 release-1.7.2 -fe7cd01828d5ca7491059f0690bb4453645eb28b release-1.7.3 -cbb146b120296852e781079d5138b04495bab6df release-1.7.4 -fe129aa02db9001d220f1db7c3c056f79482c111 release-1.7.5 -a8d111bb68847f61d682a3c8792fecb2e52efa2c release-1.7.6 -6d2fbc30f8a7f70136cf08f32d5ff3179d524873 release-1.7.7 -d5ea659b8bab2d6402a2266efa691f705e84001e release-1.7.8 -34b201c1abd1e2d4faeae4650a21574771a03c0e release-1.7.9 -860cfbcc4606ee36d898a9cd0c5ae8858db984d6 release-1.7.10 -2b3b737b5456c05cd63d3d834f4fb4d3776953d0 release-1.7.11 -3ef00a71f56420a9c3e9cec311c9a2109a015d67 release-1.7.12 -53d850fe292f157d2fb999c52788ec1dc53c91ed release-1.9.0 -884a967c369f73ab16ea859670d690fb094d3850 release-1.9.1 -3a32d6e7404a79a0973bcd8d0b83181c5bf66074 release-1.9.2 -e27a215601292872f545a733859e06d01af1017d release-1.9.3 -5cb7e2eed2031e32d2e5422caf9402758c38a6ad release-1.9.4 -942475e10cb47654205ede7ccbe7d568698e665b release-1.9.5 -b78018cfaa2f0ec20494fccb16252daa87c48a31 release-1.9.6 -54117529e40b988590ea2d38aae909b0b191663f release-1.9.7 -1bdc497c81607d854e3edf8b9a3be324c3d136b6 release-1.9.8 -ef107f3ddc237a3007e2769ec04adde0dcf627fa release-1.9.9 -be00ca08e41a69e585b6aff70a725ed6c9e1a876 release-1.9.10 -fe66cff450a95beed36a2515210eb2d7ef62c9d3 release-1.9.11 -ead3907d74f90a14d1646f1b2b56ba01d3d11702 release-1.9.12 -5936b7ed929237f1a73b467f662611cdc0309e51 release-1.9.13 -4106db71cbcb9c8274700199ac17e520902c6c0f release-1.9.14 -13070ecfda67397985f0e986eb9c42ecb46d05b5 release-1.9.15 -271ee30c6791847980cd139d31807541f5e569bf release-1.11.0 -cb783d9cc19761e14e1285d91c38f4b84d0b8756 release-1.11.1 -4d3b3a13a8cf5fc3351a7f167d1c13325e00f21c release-1.11.2 -b83a067949a3384a49fd3d943eb8d0997b31f87b release-1.11.3 -953512ca02c6f63b4fcbbc3e10d0d9835896bf99 release-1.11.4 -5253015a339aaca0a3111473d3e931b6d4752393 release-1.11.5 -5e371426b3bcba4312ce08606194b89b758927d1 release-1.11.6 -5c8f60faf33ca8926473d2da27b4c3c417bd4630 release-1.11.7 -4591da489a30f790def29bc5987f43409b503cae release-1.11.8 -20a45c768e5ed26b740679d0e22045c98727c3cc release-1.11.9 -1ad0999a7ded3d4fb01c7acf8ff57c80b643da7e release-1.11.10 -d8b321a876d6254e9e98795e3b194ef053290354 release-1.11.11 -7f394e433f0003222aa6531931ecc0b24740d5e4 release-1.11.12 -3d0e8655f897959e48cc74e87670bb5492a58871 release-1.11.13 -3671096a45bce570a2afa20b9faf42c7fb0f7e66 release-1.13.0 -539f7893ecb96bee60965528c8958d7eb2f1ce6b release-1.13.1 -5be2b25bdc65775a85f18f68a4be4f58c7384415 release-1.13.2 -8457ce87640f9bfe6221c4ac4466ced20e03bebe release-1.13.3 -bbc642c813c829963ce8197c0ca237ab7601f3d4 release-1.13.4 -0d45b4cf7c2e4e626a5a16e1fe604402ace1cea5 release-1.13.5 -f87da7d9ca02b8ced4caa6c5eb9013ccd47b0117 release-1.13.6 -47cca243d0ed39bf5dcb9859184affc958b79b6f release-1.13.7 -20ca4bcff108d3e66977f4d97508637093492287 release-1.13.8 -fb1212c7eca4c5328fe17d6cd95b010c67336aac release-1.13.9 -31c929e16910c38492581ef474e72fa67c28f124 release-1.13.10 -64179f242cb55fc206bca59de9bfdc4cf5ebcec7 release-1.13.11 -051e5fa03b92b8a564f6b12debd483d267391e82 release-1.13.12 -990b3e885636d763b97ed02d0d2cfc161a4e0c09 release-1.15.0 -4189160cb946bb38d0bc0a452b5eb4cdd8979fb5 release-1.15.1 -b234199c7ed8a156a6bb98f7ff58302c857c954f release-1.15.2 -28b3e17ca7eba1e6a0891afde0e4bc5bcc99c861 release-1.15.3 -49d49835653857daa418e68d6cbfed4958c78fca release-1.15.4 -f062e43d74fc2578bb100a9e82a953efa1eb9e4e release-1.15.5 -2351853ce6867b6166823bdf94333c0a76633c0a release-1.15.6 -051a039ce1c7e09144de4a4846669ec7116cecea release-1.15.7 -ee551e3f6dba336c0d875e266d7d55385f379b42 release-1.15.8 -d2fd76709909767fc727a5b4affcf1dc9ca488a7 release-1.15.9 -75f5c7f628411c79c7044102049f7ab4f7a246e7 release-1.15.10 -5155d0296a5ef9841f035920527ffdb771076b44 release-1.15.11 -0130ca3d58437b3c7c707cdddd813d530c68da9a release-1.15.12 -054c1c46395caff79bb4caf16f40b331f71bb6dd release-1.17.0 -7816bd7dabf6ee86c53c073b90a7143161546e06 release-1.17.1 -2fc9f853a6b7cd29dc84e0af2ed3cf78e0da6ca8 release-1.17.2 -ed4303aa1b31a9aad5440640c0840d9d0af45fed release-1.17.3 -ce2ced3856909f36f8130c99eaa4dbdbae636ddc release-1.17.4 -9af0dddbddb2c368bfedd2801bc100ffad01e19b release-1.17.5 -de68d0d94320cbf033599c6f3ca37e5335c67fd7 release-1.17.6 -e56295fe0ea76bf53b06bffa77a2d3a9a335cb8c release-1.17.7 -fdacd273711ddf20f778c1fb91529ab53979a454 release-1.17.8 -5e8d52bca714d4b85284ddb649d1ba4a3ca978a8 release-1.17.9 -c44970de01474f6f3e01b0adea85ec1d03e3a5f2 release-1.17.10 -cbe6ba650211541310618849168631ce0b788f35 release-1.19.0 -062920e2f3bf871ef7a3d8496edec1b3065faf80 release-1.19.1 -a7b46539f507e6c64efa0efda69ad60b6f4ffbce release-1.19.2 -3cbc2602325f0ac08917a4397d76f5155c34b7b1 release-1.19.3 -dc0cc425fa63a80315f6efb68697cadb6626cdf2 release-1.19.4 -8e5b068f761cd512d10c9671fbde0b568c1fd08b release-1.19.5 -f618488eb769e0ed74ef0d93cd118d2ad79ef94d release-1.19.6 -3fa6e2095a7a51acc630517e1c27a7b7ac41f7b3 release-1.19.7 -8c65d21464aaa5923775f80c32474adc7a320068 release-1.19.8 -da571b8eaf8f30f36c43b3c9b25e01e31f47149c release-1.19.9 -ffcbb9980ee2bad27b4d7b1cd680b14ff47b29aa release-1.19.10 -df34dcc9ac072ffd0945e5a1f3eb7987e8275375 release-1.21.0 -a68ac0677f8553b1f84d357bc9da114731ab5f47 release-1.21.1 -bfbc52374adcbf2f9060afd62de940f6fab3bba5 release-1.21.2 -2217a9c1d0b86026f22700b3c089545db1964f55 release-1.21.3 -39be8a682c58308d9399cddd57e37f9fdb7bdf3e release-1.21.4 -d986378168fd4d70e0121cabac274c560cca9bdf release-1.21.5 -714eb4b2c09e712fb2572a2164ce2bf67638ccac release-1.21.6 -5da2c0902e8e2aa4534008a582a60c61c135960e release-1.23.0 -a63d0a70afea96813ba6667997bc7d68b5863f0d release-1.23.1 -aa901551a7ebad1e8b0f8c11cb44e3424ba29707 release-1.23.2 -ff3afd1ce6a6b65057741df442adfaa71a0e2588 release-1.23.3 -ac779115ed6ee4f3039e9aea414a54e560450ee2 release-1.23.4 -12dcf92b0c2c68552398f19644ce3104459807d7 release-1.25.0 -f8134640e8615448205785cf00b0bc810489b495 release-1.25.1 -1d839f05409d1a50d0f15a2bf36547001f99ae40 release-1.25.2 -294a3d07234f8f65d7b0e0b0e2c5b05c12c5da0a release-1.25.3 -173a0a7dbce569adbb70257c6ec4f0f6bc585009 release-1.25.4 -8618e4d900cc71082fbe7dc72af087937d64faf5 release-1.25.5 -2166e329fb4ed7d6da7c823ee6499f7d06d7bc00 release-1.27.0 -417b1045b18ecbca0ca583073a221ffbfbaee5d9 release-1.27.1 From 81a933e1f6fa26012d6736006456171e1f1b8977 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Thu, 29 Aug 2024 16:03:58 +0400 Subject: [PATCH 155/279] Switched GNUmakefile from hg to git. --- misc/GNUmakefile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/misc/GNUmakefile b/misc/GNUmakefile index fa4c36d49f0..7d4b657de1d 100644 --- a/misc/GNUmakefile +++ b/misc/GNUmakefile @@ -30,12 +30,12 @@ release: export export: rm -rf $(TEMP) - hg archive -X '.hg*' $(TEMP)/$(NGINX) + git archive --prefix=$(TEMP)/$(NGINX)/ HEAD | tar -x -f - --exclude '.git*' RELEASE: - hg ci -m nginx-$(VER)-RELEASE - hg tag -m "release-$(VER) tag" release-$(VER) + git commit -m nginx-$(VER)-RELEASE + git tag -m "release-$(VER) tag" release-$(VER) $(MAKE) -f misc/GNUmakefile release @@ -93,8 +93,8 @@ zip: export sed -i '' -e "s/$$/`printf '\r'`/" $(TEMP)/$(NGINX)/conf/* - mv $(TEMP)/$(NGINX)/docs/text/LICENSE $(TEMP)/$(NGINX)/docs.new - mv $(TEMP)/$(NGINX)/docs/text/README $(TEMP)/$(NGINX)/docs.new + mv $(TEMP)/$(NGINX)/LICENSE $(TEMP)/$(NGINX)/docs.new + mv $(TEMP)/$(NGINX)/README $(TEMP)/$(NGINX)/docs.new mv $(TEMP)/$(NGINX)/docs/html $(TEMP)/$(NGINX) rm -r $(TEMP)/$(NGINX)/docs From 863ab647cd659d6b326c61a1f1227629978ce132 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Thu, 29 Aug 2024 16:24:03 +0400 Subject: [PATCH 156/279] Moved LICENSE and README to root. --- docs/text/LICENSE => LICENSE | 0 docs/text/README => README | 0 misc/GNUmakefile | 2 -- 3 files changed, 2 deletions(-) rename docs/text/LICENSE => LICENSE (100%) rename docs/text/README => README (100%) diff --git a/docs/text/LICENSE b/LICENSE similarity index 100% rename from docs/text/LICENSE rename to LICENSE diff --git a/docs/text/README b/README similarity index 100% rename from docs/text/README rename to README diff --git a/misc/GNUmakefile b/misc/GNUmakefile index 7d4b657de1d..4b6a1275916 100644 --- a/misc/GNUmakefile +++ b/misc/GNUmakefile @@ -15,8 +15,6 @@ release: export mv $(TEMP)/$(NGINX)/auto/configure $(TEMP)/$(NGINX) - mv $(TEMP)/$(NGINX)/docs/text/LICENSE $(TEMP)/$(NGINX) - mv $(TEMP)/$(NGINX)/docs/text/README $(TEMP)/$(NGINX) mv $(TEMP)/$(NGINX)/docs/html $(TEMP)/$(NGINX) mv $(TEMP)/$(NGINX)/docs/man $(TEMP)/$(NGINX) From 6bb4be1a79e8c8e8a07f4df1c77a80c65b3b6f47 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Thu, 29 Aug 2024 17:14:25 +0400 Subject: [PATCH 157/279] Removed C-style comments from LICENSE. --- LICENSE | 50 ++++++++++++++++++++++++-------------------------- 1 file changed, 24 insertions(+), 26 deletions(-) diff --git a/LICENSE b/LICENSE index 985470ef99c..a7ec58a754e 100644 --- a/LICENSE +++ b/LICENSE @@ -1,26 +1,24 @@ -/* - * Copyright (C) 2002-2021 Igor Sysoev - * Copyright (C) 2011-2024 Nginx, Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ +Copyright (C) 2002-2021 Igor Sysoev +Copyright (C) 2011-2024 Nginx, Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +SUCH DAMAGE. From e73ac62294ae85977dde1e8159857e6a4d227f26 Mon Sep 17 00:00:00 2001 From: Maryna Herasimovich Date: Wed, 28 Aug 2024 20:13:13 -0700 Subject: [PATCH 158/279] Added Code of Conduct. --- CODE_OF_CONDUCT.md | 126 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 CODE_OF_CONDUCT.md diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 00000000000..a99d8854a60 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,126 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio economic status, +nationality, personal appearance, race, caste, color, religion, or sexual +identity and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +- Demonstrating empathy and kindness toward other people +- Being respectful of differing opinions, viewpoints, and experiences +- Giving and gracefully accepting constructive feedback +- Accepting responsibility and apologizing to those affected by our mistakes +and learning from the experience +- Focusing on what is best not just for us as individuals, but for the +overall community + +Examples of unacceptable behavior include: + +- The use of sexualized language or imagery, and sexual attention or advances +of any kind +- Trolling, insulting or derogatory comments, and personal or political attacks +- Public or private harassment +- Publishing others' private information, such as a physical or email address, +without their explicit permission +- Other conduct which could reasonably be considered inappropriate in a +professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards +of acceptable behavior and will take appropriate and fair corrective action +in response to any behavior that they deem inappropriate, threatening, +offensive, or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for +moderation decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official email address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +. All complaints will be reviewed and investigated +promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series of actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external +channels like social media. Violating these terms may lead to a temporary or +permanent ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within +the community. + +## Attribution + +This Code of Conduct is adapted from the +[Contributor Covenant](https://www.contributor-covenant.org), version 2.1, +available at +. + +Community Impact Guidelines were inspired by +[Mozilla's code of conduct enforcement ladder](https://github.com/mozilla/inclusion). + +For answers to common questions about this code of conduct, see the FAQ at +. Translations are available at +. From 3b16b46aae979b9a130c5dd42430dda37b623282 Mon Sep 17 00:00:00 2001 From: Maryna Herasimovich Date: Wed, 28 Aug 2024 20:51:54 -0700 Subject: [PATCH 159/279] Added security policy. --- SECURITY.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 SECURITY.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000000..2b48e47e3ae --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,20 @@ +# Security Policy + +## Latest Versions + +We advise users to run the most recent mainline or stable release of nginx. + +## Reporting a Vulnerability + +Please report any vulnerabilities via one of the following methods +(in order of preference): + +1. [Report a vulnerability](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing-information-about-vulnerabilities/privately-reporting-a-security-vulnerability) +within this repository. We are using the Github workflow that allows us to +manage vulnerabilities in a private manner and to interact with reporters +securely. + +2. [Report directly to F5](https://www.f5.com/services/support/report-a-vulnerability). + +3. Report via email to security-alert@nginx.org. +This method will be deprecated in the future. From da468ec0c03ab7711ab4fcb0517760daaf31ab10 Mon Sep 17 00:00:00 2001 From: Maryna Herasimovich Date: Wed, 28 Aug 2024 20:43:08 -0700 Subject: [PATCH 160/279] Added contributing guidelines. --- CONTRIBUTING.md | 110 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000000..864e7989b15 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,110 @@ +# Contributing Guidelines + +The following is a set of guidelines for contributing to nginx project. +We really appreciate that you are considering contributing! + +## Table of Contents + +- [Ask a Question](#ask-a-question) +- [Report a Bug](#report-a-bug) +- [Suggest a Feature or Enhancement](#suggest-a-feature-or-enhancement) +- [Open a Discussion](#open-a-discussion) +- [Submit a Pull Request](#submit-a-pull-request) +- [Issue Lifecycle](#issue-lifecycle) + +## Ask a Question + +To ask a question, open an issue on GitHub with the label `question`. + +## Report a Bug + +To report a bug, open an issue on GitHub with the label `bug` using the +available bug report issue template. Before reporting a bug, make sure the +issue has not already been reported. + +## Suggest a Feature or Enhancement + +To suggest a feature or enhancement, open an issue on GitHub with the label +`feature` or `enhancement` using the available feature request issue template. +Please ensure the feature or enhancement has not already been suggested. + +## Open a Discussion + +If you want to engage in a conversation with the community and maintainers, +we encourage you to use +[GitHub Discussions](https://github.com/nginx/nginx/discussions). + +## Submit a Pull Request + +Follow this plan to contribute a change to NGINX source code: + +- Fork the NGINX repository +- Create a branch +- Implement your changes in this branch +- Submit a pull request (PR) when your changes are tested and ready for review + +Refer to +[NGINX Development Guide](https://nginx.org/en/docs/dev/development_guide.html) +for questions about NGINX programming. + +### Formatting Changes + +- Changes should be formatted according to the +[code style](https://nginx.org/en/docs/dev/development_guide.html#code_style) +used by NGINX; sometimes, there is no clear rule, in which case examine how +existing NGINX sources are formatted and mimic this style; changes will more +likely be accepted if style corresponds to the surrounding code + +- Keep a clean, concise and meaningful commit history on your branch, rebasing +locally and breaking changes logically into commits before submitting a PR + +- Each commit message should have a single-line subject line followed by verbose +description after an empty line + +- Limit the subject line to 67 characters, and the rest of the commit message +to 76 characters + +- Use subject line prefixes for commits that affect a specific portion of the +code; examples include "Upstream:", "QUIC:", or "Core:"; see the commit history +to get an idea of the prefixes used + +- Reference issues in the the subject line; if the commit fixes an issue, +[name it](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue) +accordingly + +### Before Submitting + +- The proposed changes should work properly on a wide range of +[supported platforms](https://nginx.org/en/index.html#tested_os_and_platforms) + +- Try to make it clear why the suggested change is needed, and provide a use +case, if possible + +- Passing your changes through the test suite is a good way to ensure that they +do not cause a regression; the repository with tests can be cloned with the +following command: + +```bash +git clone https://github.com/nginx/nginx-tests.git +``` + +- Submitting a change implies granting project a permission to use it under the +[BSD-2-Clause license](https://github.com/nginx/nginx/blob/master/LICENSE) + +## Issue Lifecycle + +To ensure a balance between work carried out by the NGINX engineering team +while encouraging community involvement on this project, we use the following +issue lifecycle: + +- A new issue is created by a community member + +- An owner on the NGINX engineering team is assigned to the issue; this +owner shepherds the issue through the subsequent stages in the issue lifecycle + +- The owner assigns one or more +[labels](https://github.com/nginx/nginx/issues/labels) to the issue + +- The owner, in collaboration with the wider team (product management and +engineering), determines what milestone to attach to an issue; +generally, milestones correspond to product releases From 082a3cbe3b7f6c28630f271c74b1787220974ed9 Mon Sep 17 00:00:00 2001 From: Maryna Herasimovich Date: Thu, 29 Aug 2024 13:06:48 -0700 Subject: [PATCH 161/279] Added GitHub templates. --- .github/ISSUE_TEMPLATE/bug_report.md | 38 +++++++++++++++++++++++ .github/ISSUE_TEMPLATE/feature_request.md | 18 +++++++++++ .github/pull_request_template.md | 10 ++++++ 3 files changed, 66 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/pull_request_template.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 00000000000..efdc1167f07 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,38 @@ +--- +name: Bug report +about: Create a report to help us improve +title: "" +labels: "bug" +--- + +### Environment + +Include the result of the following commands: + - `nginx -V` + - `uname -a` + +### Description + +Describe the bug in full detail including expected and actual behavior. +Specify conditions that caused it. Provide the relevant part of nginx +configuration and debug log. + +- [ ] The bug is reproducible with the latest version of nginx +- [ ] The nginx configuration is minimized to the smallest possible +to reproduce the issue and doesn't contain third-party modules + +#### nginx configuration + +``` +# Your nginx configuration here +``` +or share the configuration in [gist](https://gist.github.com/). + +#### nginx debug log + +It is advised to enable +[debug logging](http://nginx.org/en/docs/debugging_log.html). +``` +# Your nginx debug log here +``` +or share the debug log in [gist](https://gist.github.com/). diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 00000000000..67471b08302 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,18 @@ +--- +name: Feature request +about: Suggest a feature for nginx +title: "" +labels: "feature" +--- + +### Describe the feature you'd like to add to nginx + +A clear and concise description of the feature. + +### Describe the problem this feature solves + +A clear and concise description of the problem. + +### Additional context + +Add any other context about the feature request here. diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 00000000000..0ee1fc3643f --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,10 @@ +### Proposed changes + +Describe the use case and detail of the change. + +If this pull request addresses an issue on GitHub, make sure to reference that +issue using one of the +[supported keywords](https://docs.github.com/en/github/managing-your-work-on-github/linking-a-pull-request-to-an-issue). + +Before creating a pull request, make sure to comply with +[`Contributing Guidelines`](/CONTRIBUTING.md). From 042b9cc4db564dc16dfc93c31c14e9a6bb749509 Mon Sep 17 00:00:00 2001 From: Konstantin Pavlov Date: Thu, 18 Jul 2024 13:42:43 -0700 Subject: [PATCH 162/279] Added CI based on GitHub Actions. Pushes to master and stable branches will result in buildbot-like checks on multiple OSes and architectures. Pull requests will be checked on a public Ubuntu GitHub runner. --- .github/workflows/buildbot.yml | 11 +++++++++++ .github/workflows/check-pr.yml | 8 ++++++++ 2 files changed, 19 insertions(+) create mode 100644 .github/workflows/buildbot.yml create mode 100644 .github/workflows/check-pr.yml diff --git a/.github/workflows/buildbot.yml b/.github/workflows/buildbot.yml new file mode 100644 index 00000000000..484b74f985f --- /dev/null +++ b/.github/workflows/buildbot.yml @@ -0,0 +1,11 @@ +name: buildbot + +on: + push: + branches: + - master + - 'stable-1.*' + +jobs: + buildbot: + uses: nginx/ci-self-hosted/.github/workflows/nginx-buildbot.yml@main diff --git a/.github/workflows/check-pr.yml b/.github/workflows/check-pr.yml new file mode 100644 index 00000000000..92c0ea6bfe1 --- /dev/null +++ b/.github/workflows/check-pr.yml @@ -0,0 +1,8 @@ +name: check-pr + +on: + pull_request: + +jobs: + check-pr: + uses: nginx/ci-self-hosted/.github/workflows/nginx-check-pr.yml@main From 00637cce366f17b78fe1ed5c1ef0e534143045f6 Mon Sep 17 00:00:00 2001 From: Shaikh Yaser Date: Thu, 5 Sep 2024 20:47:54 +0530 Subject: [PATCH 163/279] Fixed a typo in win-utf. --- conf/win-utf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/win-utf b/conf/win-utf index ed8bc007a72..d0b7116c8a3 100644 --- a/conf/win-utf +++ b/conf/win-utf @@ -37,7 +37,7 @@ charset_map windows-1251 utf-8 { AA D084 ; # capital Ukrainian YE AB C2AB ; # left-pointing double angle quotation mark AC C2AC ; # not sign - AD C2AD ; # soft hypen + AD C2AD ; # soft hyphen AE C2AE ; # (R) AF D087 ; # capital Ukrainian YI From 1a64c196a7d43f83a14fec20ce8936e599c92865 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Tue, 10 Sep 2024 16:48:11 +0400 Subject: [PATCH 164/279] Proxy: proxy_pass_trailers directive. The directive allows to pass upstream response trailers to client. --- src/http/modules/ngx_http_proxy_module.c | 233 +++++++++++++++++++++-- src/http/ngx_http.h | 2 +- src/http/ngx_http_parse.c | 11 +- src/http/ngx_http_request_body.c | 4 +- src/http/ngx_http_upstream.h | 2 +- 5 files changed, 235 insertions(+), 17 deletions(-) diff --git a/src/http/modules/ngx_http_proxy_module.c b/src/http/modules/ngx_http_proxy_module.c index b774c866e27..f9a373744ad 100644 --- a/src/http/modules/ngx_http_proxy_module.c +++ b/src/http/modules/ngx_http_proxy_module.c @@ -138,6 +138,8 @@ typedef struct { ngx_chain_t *free; ngx_chain_t *busy; + ngx_buf_t *trailers; + unsigned head:1; unsigned internal_chunked:1; unsigned header_sent:1; @@ -163,6 +165,8 @@ static ngx_int_t ngx_http_proxy_non_buffered_copy_filter(void *data, ssize_t bytes); static ngx_int_t ngx_http_proxy_non_buffered_chunked_filter(void *data, ssize_t bytes); +static ngx_int_t ngx_http_proxy_process_trailer(ngx_http_request_t *r, + ngx_buf_t *buf); static void ngx_http_proxy_abort_request(ngx_http_request_t *r); static void ngx_http_proxy_finalize_request(ngx_http_request_t *r, ngx_int_t rc); @@ -457,6 +461,13 @@ static ngx_command_t ngx_http_proxy_commands[] = { offsetof(ngx_http_proxy_loc_conf_t, upstream.pass_request_body), NULL }, + { ngx_string("proxy_pass_trailers"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG, + ngx_conf_set_flag_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_proxy_loc_conf_t, upstream.pass_trailers), + NULL }, + { ngx_string("proxy_buffer_size"), NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, ngx_conf_set_size_slot, @@ -2181,11 +2192,12 @@ ngx_http_proxy_copy_filter(ngx_event_pipe_t *p, ngx_buf_t *buf) static ngx_int_t ngx_http_proxy_chunked_filter(ngx_event_pipe_t *p, ngx_buf_t *buf) { - ngx_int_t rc; - ngx_buf_t *b, **prev; - ngx_chain_t *cl; - ngx_http_request_t *r; - ngx_http_proxy_ctx_t *ctx; + ngx_int_t rc; + ngx_buf_t *b, **prev; + ngx_chain_t *cl; + ngx_http_request_t *r; + ngx_http_proxy_ctx_t *ctx; + ngx_http_proxy_loc_conf_t *plcf; if (buf->pos == buf->last) { return NGX_OK; @@ -2216,11 +2228,39 @@ ngx_http_proxy_chunked_filter(ngx_event_pipe_t *p, ngx_buf_t *buf) } b = NULL; + + if (ctx->trailers) { + rc = ngx_http_proxy_process_trailer(r, buf); + + if (rc == NGX_ERROR) { + return NGX_ERROR; + } + + if (rc == NGX_OK) { + + /* a whole response has been parsed successfully */ + + p->length = 0; + r->upstream->keepalive = !r->upstream->headers_in.connection_close; + + if (buf->pos != buf->last) { + ngx_log_error(NGX_LOG_WARN, p->log, 0, + "upstream sent data after trailers"); + r->upstream->keepalive = 0; + } + } + + goto free_buf; + } + + plcf = ngx_http_get_module_loc_conf(r, ngx_http_proxy_module); + prev = &buf->shadow; for ( ;; ) { - rc = ngx_http_parse_chunked(r, buf, &ctx->chunked); + rc = ngx_http_parse_chunked(r, buf, &ctx->chunked, + plcf->upstream.pass_trailers); if (rc == NGX_OK) { @@ -2275,6 +2315,19 @@ ngx_http_proxy_chunked_filter(ngx_event_pipe_t *p, ngx_buf_t *buf) if (rc == NGX_DONE) { + if (plcf->upstream.pass_trailers) { + rc = ngx_http_proxy_process_trailer(r, buf); + + if (rc == NGX_ERROR) { + return NGX_ERROR; + } + + if (rc == NGX_AGAIN) { + p->length = 1; + break; + } + } + /* a whole response has been parsed successfully */ p->length = 0; @@ -2306,6 +2359,8 @@ ngx_http_proxy_chunked_filter(ngx_event_pipe_t *p, ngx_buf_t *buf) return NGX_ERROR; } +free_buf: + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, p->log, 0, "http proxy chunked state %ui, length %O", ctx->chunked.state, p->length); @@ -2401,11 +2456,14 @@ ngx_http_proxy_non_buffered_chunked_filter(void *data, ssize_t bytes) { ngx_http_request_t *r = data; - ngx_int_t rc; - ngx_buf_t *b, *buf; - ngx_chain_t *cl, **ll; - ngx_http_upstream_t *u; - ngx_http_proxy_ctx_t *ctx; + ngx_int_t rc; + ngx_buf_t *b, *buf; + ngx_chain_t *cl, **ll; + ngx_http_upstream_t *u; + ngx_http_proxy_ctx_t *ctx; + ngx_http_proxy_loc_conf_t *plcf; + + plcf = ngx_http_get_module_loc_conf(r, ngx_http_proxy_module); ctx = ngx_http_get_module_ctx(r, ngx_http_proxy_module); @@ -2419,13 +2477,38 @@ ngx_http_proxy_non_buffered_chunked_filter(void *data, ssize_t bytes) buf->pos = buf->last; buf->last += bytes; + if (ctx->trailers) { + rc = ngx_http_proxy_process_trailer(r, buf); + + if (rc == NGX_ERROR) { + return NGX_ERROR; + } + + if (rc == NGX_OK) { + + /* a whole response has been parsed successfully */ + + r->upstream->keepalive = !u->headers_in.connection_close; + u->length = 0; + + if (buf->pos != buf->last) { + ngx_log_error(NGX_LOG_WARN, r->connection->log, 0, + "upstream sent data after trailers"); + u->keepalive = 0; + } + } + + return NGX_OK; + } + for (cl = u->out_bufs, ll = &u->out_bufs; cl; cl = cl->next) { ll = &cl->next; } for ( ;; ) { - rc = ngx_http_parse_chunked(r, buf, &ctx->chunked); + rc = ngx_http_parse_chunked(r, buf, &ctx->chunked, + plcf->upstream.pass_trailers); if (rc == NGX_OK) { @@ -2467,6 +2550,19 @@ ngx_http_proxy_non_buffered_chunked_filter(void *data, ssize_t bytes) if (rc == NGX_DONE) { + if (plcf->upstream.pass_trailers) { + rc = ngx_http_proxy_process_trailer(r, buf); + + if (rc == NGX_ERROR) { + return NGX_ERROR; + } + + if (rc == NGX_AGAIN) { + u->length = 1; + break; + } + } + /* a whole response has been parsed successfully */ u->keepalive = !u->headers_in.connection_close; @@ -2497,6 +2593,115 @@ ngx_http_proxy_non_buffered_chunked_filter(void *data, ssize_t bytes) } +static ngx_int_t +ngx_http_proxy_process_trailer(ngx_http_request_t *r, ngx_buf_t *buf) +{ + size_t len; + ngx_int_t rc; + ngx_buf_t *b; + ngx_table_elt_t *h; + ngx_http_proxy_ctx_t *ctx; + ngx_http_proxy_loc_conf_t *plcf; + + plcf = ngx_http_get_module_loc_conf(r, ngx_http_proxy_module); + + ctx = ngx_http_get_module_ctx(r, ngx_http_proxy_module); + + if (ctx->trailers == NULL) { + ctx->trailers = ngx_create_temp_buf(r->pool, + plcf->upstream.buffer_size); + if (ctx->trailers == NULL) { + return NGX_ERROR; + } + } + + b = ctx->trailers; + len = ngx_min(buf->last - buf->pos, b->end - b->last); + + b->last = ngx_cpymem(b->last, buf->pos, len); + + for ( ;; ) { + + rc = ngx_http_parse_header_line(r, b, 1); + + if (rc == NGX_OK) { + + /* a header line has been parsed successfully */ + + h = ngx_list_push(&r->upstream->headers_in.trailers); + if (h == NULL) { + return NGX_ERROR; + } + + h->hash = r->header_hash; + + h->key.len = r->header_name_end - r->header_name_start; + h->value.len = r->header_end - r->header_start; + + h->key.data = ngx_pnalloc(r->pool, + h->key.len + 1 + h->value.len + 1 + h->key.len); + if (h->key.data == NULL) { + h->hash = 0; + return NGX_ERROR; + } + + h->value.data = h->key.data + h->key.len + 1; + h->lowcase_key = h->key.data + h->key.len + 1 + h->value.len + 1; + + ngx_memcpy(h->key.data, r->header_name_start, h->key.len); + h->key.data[h->key.len] = '\0'; + ngx_memcpy(h->value.data, r->header_start, h->value.len); + h->value.data[h->value.len] = '\0'; + + if (h->key.len == r->lowcase_index) { + ngx_memcpy(h->lowcase_key, r->lowcase_header, h->key.len); + + } else { + ngx_strlow(h->lowcase_key, h->key.data, h->key.len); + } + + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http proxy trailer: \"%V: %V\"", + &h->key, &h->value); + continue; + } + + if (rc == NGX_HTTP_PARSE_HEADER_DONE) { + + /* a whole header has been parsed successfully */ + + buf->pos += len - (b->last - b->pos); + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http proxy trailer done"); + + return NGX_OK; + } + + if (rc == NGX_AGAIN) { + buf->pos += len; + + if (b->last == b->end) { + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "upstream sent too big trailers"); + return NGX_ERROR; + } + + return NGX_AGAIN; + } + + /* rc == NGX_HTTP_PARSE_INVALID_HEADER */ + + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "upstream sent invalid trailer: \"%*s\\x%02xd...\"", + r->header_end - r->header_name_start, + r->header_name_start, *r->header_end); + + return NGX_ERROR; + } +} + + static void ngx_http_proxy_abort_request(ngx_http_request_t *r) { @@ -3379,6 +3584,7 @@ ngx_http_proxy_create_loc_conf(ngx_conf_t *cf) conf->upstream.pass_request_headers = NGX_CONF_UNSET; conf->upstream.pass_request_body = NGX_CONF_UNSET; + conf->upstream.pass_trailers = NGX_CONF_UNSET; #if (NGX_HTTP_CACHE) conf->upstream.cache = NGX_CONF_UNSET; @@ -3721,6 +3927,9 @@ ngx_http_proxy_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) ngx_conf_merge_value(conf->upstream.pass_request_body, prev->upstream.pass_request_body, 1); + ngx_conf_merge_value(conf->upstream.pass_trailers, + prev->upstream.pass_trailers, 0); + ngx_conf_merge_value(conf->upstream.intercept_errors, prev->upstream.intercept_errors, 0); diff --git a/src/http/ngx_http.h b/src/http/ngx_http.h index e06464ebd9e..cb4a1e68a28 100644 --- a/src/http/ngx_http.h +++ b/src/http/ngx_http.h @@ -117,7 +117,7 @@ ngx_int_t ngx_http_arg(ngx_http_request_t *r, u_char *name, size_t len, void ngx_http_split_args(ngx_http_request_t *r, ngx_str_t *uri, ngx_str_t *args); ngx_int_t ngx_http_parse_chunked(ngx_http_request_t *r, ngx_buf_t *b, - ngx_http_chunked_t *ctx); + ngx_http_chunked_t *ctx, ngx_uint_t keep_trailers); ngx_http_request_t *ngx_http_create_request(ngx_connection_t *c); diff --git a/src/http/ngx_http_parse.c b/src/http/ngx_http_parse.c index f7e50388f99..a45c04554b4 100644 --- a/src/http/ngx_http_parse.c +++ b/src/http/ngx_http_parse.c @@ -2140,7 +2140,7 @@ ngx_http_split_args(ngx_http_request_t *r, ngx_str_t *uri, ngx_str_t *args) ngx_int_t ngx_http_parse_chunked(ngx_http_request_t *r, ngx_buf_t *b, - ngx_http_chunked_t *ctx) + ngx_http_chunked_t *ctx, ngx_uint_t keep_trailers) { u_char *pos, ch, c; ngx_int_t rc; @@ -2218,6 +2218,9 @@ ngx_http_parse_chunked(ngx_http_request_t *r, ngx_buf_t *b, state = sw_last_chunk_extension_almost_done; break; case LF: + if (keep_trailers) { + goto done; + } state = sw_trailer; break; case ';': @@ -2297,12 +2300,18 @@ ngx_http_parse_chunked(ngx_http_request_t *r, ngx_buf_t *b, state = sw_last_chunk_extension_almost_done; break; case LF: + if (keep_trailers) { + goto done; + } state = sw_trailer; } break; case sw_last_chunk_extension_almost_done: if (ch == LF) { + if (keep_trailers) { + goto done; + } state = sw_trailer; break; } diff --git a/src/http/ngx_http_request_body.c b/src/http/ngx_http_request_body.c index afb04239556..93c69220c53 100644 --- a/src/http/ngx_http_request_body.c +++ b/src/http/ngx_http_request_body.c @@ -870,7 +870,7 @@ ngx_http_discard_request_body_filter(ngx_http_request_t *r, ngx_buf_t *b) for ( ;; ) { - rc = ngx_http_parse_chunked(r, b, rb->chunked); + rc = ngx_http_parse_chunked(r, b, rb->chunked, 0); if (rc == NGX_OK) { @@ -1131,7 +1131,7 @@ ngx_http_request_body_chunked_filter(ngx_http_request_t *r, ngx_chain_t *in) cl->buf->file_pos, cl->buf->file_last - cl->buf->file_pos); - rc = ngx_http_parse_chunked(r, cl->buf, rb->chunked); + rc = ngx_http_parse_chunked(r, cl->buf, rb->chunked, 0); if (rc == NGX_OK) { diff --git a/src/http/ngx_http_upstream.h b/src/http/ngx_http_upstream.h index cb16f2b4d5b..db0e1424e11 100644 --- a/src/http/ngx_http_upstream.h +++ b/src/http/ngx_http_upstream.h @@ -176,6 +176,7 @@ typedef struct { ngx_flag_t request_buffering; ngx_flag_t pass_request_headers; ngx_flag_t pass_request_body; + ngx_flag_t pass_trailers; ngx_flag_t ignore_client_abort; ngx_flag_t intercept_errors; @@ -224,7 +225,6 @@ typedef struct { signed store:2; unsigned intercept_404:1; unsigned change_buffering:1; - unsigned pass_trailers:1; unsigned preserve_output:1; #if (NGX_HTTP_SSL || NGX_COMPAT) From 18afcda938cd2d4712d0d083b57161290a5a2d34 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Fri, 20 Sep 2024 14:08:42 +0400 Subject: [PATCH 165/279] SSL: optional ssl_client_certificate for ssl_verify_client. Starting from TLSv1.1 (as seen since draft-ietf-tls-rfc2246-bis-00), the "certificate_authorities" field grammar of the CertificateRequest message was redone to allow no distinguished names. In TLSv1.3, with the restructured CertificateRequest message, this can be similarly done by optionally including the "certificate_authorities" extension. This allows to avoid sending DNs at all. In practice, aside from published TLS specifications, all supported SSL/TLS libraries allow to request client certificates with an empty DN list for any protocol version. For instance, when operating in TLSv1, this results in sending the "certificate_authorities" list as a zero-length vector, which corresponds to the TLSv1.1 specification. Such behaviour goes back to SSLeay. The change relaxes the requirement to specify at least one trusted CA certificate in the ssl_client_certificate directive, which resulted in sending DNs of these certificates (closes #142). Instead, all trusted CA certificates can be specified now using the ssl_trusted_certificate directive if needed. A notable difference that certificates specified in ssl_trusted_certificate are always loaded remains (see 3648ba7db). Co-authored-by: Praveen Chaudhary --- src/http/modules/ngx_http_ssl_module.c | 8 ++++++-- src/mail/ngx_mail_ssl_module.c | 8 ++++++-- src/stream/ngx_stream_ssl_module.c | 8 ++++++-- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/http/modules/ngx_http_ssl_module.c b/src/http/modules/ngx_http_ssl_module.c index 1c92d9fa879..abc8d49ab69 100644 --- a/src/http/modules/ngx_http_ssl_module.c +++ b/src/http/modules/ngx_http_ssl_module.c @@ -787,9 +787,13 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) if (conf->verify) { - if (conf->client_certificate.len == 0 && conf->verify != 3) { + if (conf->verify != 3 + && conf->client_certificate.len == 0 + && conf->trusted_certificate.len == 0) + { ngx_log_error(NGX_LOG_EMERG, cf->log, 0, - "no ssl_client_certificate for ssl_verify_client"); + "no ssl_client_certificate or " + "ssl_trusted_certificate for ssl_verify_client"); return NGX_CONF_ERROR; } diff --git a/src/mail/ngx_mail_ssl_module.c b/src/mail/ngx_mail_ssl_module.c index aebb4ccb6f8..b547dc1011a 100644 --- a/src/mail/ngx_mail_ssl_module.c +++ b/src/mail/ngx_mail_ssl_module.c @@ -450,9 +450,13 @@ ngx_mail_ssl_merge_conf(ngx_conf_t *cf, void *parent, void *child) if (conf->verify) { - if (conf->client_certificate.len == 0 && conf->verify != 3) { + if (conf->verify != 3 + && conf->client_certificate.len == 0 + && conf->trusted_certificate.len == 0) + { ngx_log_error(NGX_LOG_EMERG, cf->log, 0, - "no ssl_client_certificate for ssl_verify_client"); + "no ssl_client_certificate or " + "ssl_trusted_certificate for ssl_verify_client"); return NGX_CONF_ERROR; } diff --git a/src/stream/ngx_stream_ssl_module.c b/src/stream/ngx_stream_ssl_module.c index 072e74917cf..0233a9258b7 100644 --- a/src/stream/ngx_stream_ssl_module.c +++ b/src/stream/ngx_stream_ssl_module.c @@ -1008,9 +1008,13 @@ ngx_stream_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) if (conf->verify) { - if (conf->client_certificate.len == 0 && conf->verify != 3) { + if (conf->verify != 3 + && conf->client_certificate.len == 0 + && conf->trusted_certificate.len == 0) + { ngx_log_error(NGX_LOG_EMERG, cf->log, 0, - "no ssl_client_certificate for ssl_verify_client"); + "no ssl_client_certificate or " + "ssl_trusted_certificate for ssl_verify_client"); return NGX_CONF_ERROR; } From b1e07409b1071564e8dd8c70436cba5da19a1575 Mon Sep 17 00:00:00 2001 From: Michael Vernik Date: Sat, 31 Aug 2024 22:42:09 -0700 Subject: [PATCH 166/279] Added new primary README.md file. --- README | 3 - README.md | 230 +++++++++++++++++++++++++++++++++++++++++++++++ misc/GNUmakefile | 2 +- 3 files changed, 231 insertions(+), 4 deletions(-) delete mode 100644 README create mode 100644 README.md diff --git a/README b/README deleted file mode 100644 index 2f68e14e0d8..00000000000 --- a/README +++ /dev/null @@ -1,3 +0,0 @@ - -Documentation is available at http://nginx.org - diff --git a/README.md b/README.md new file mode 100644 index 00000000000..b97181e3065 --- /dev/null +++ b/README.md @@ -0,0 +1,230 @@ + + + + NGINX Banner + + +NGINX (pronounced "engine x" or "en-jin-eks") is the world's most popular Web Server, high performance Load Balancer, Reverse Proxy, API Gateway and Content Cache. + +NGINX is free and open source software, distributed under the terms of a simplified [2-clause BSD-like license](LICENSE). + +Enterprise distributions, commercial support and training are available from [F5, Inc](https://www.f5.com/products/nginx). + +> [!IMPORTANT] +> The goal of this README is to provide a basic, structured introduction to NGINX for novice users. Please refer to the [full NGINX documentation](https://nginx.org/en/docs/) for detailed information on [installing](https://nginx.org/en/docs/install.html), [building](https://nginx.org/en/docs/configure.html), [configuring](https://nginx.org/en/docs/dirindex.html), [debugging](https://nginx.org/en/docs/debugging_log.html), and more. These documentation pages also contain a more detailed [Beginners Guide](https://nginx.org/en/docs/beginners_guide.html), How-Tos, [Development guide](https://nginx.org/en/docs/dev/development_guide.html), and a complete module and [directive reference](https://nginx.org/en/docs/dirindex.html). + +# Table of contents +- [How it works](#how-it-works) + - [Modules](#modules) + - [Configurations](#configurations) + - [Runtime](#runtime) +- [Downloading and installing](#downloading-and-installing) + - [Stable and Mainline binaries](#stable-and-mainline-binaries) + - [Linux binary installation process](#linux-binary-installation-process) + - [FreeBSD installation process](#freebsd-installation-process) + - [Windows executables](#windows-executables) + - [Dynamic modules](#dynamic-modules) +- [Getting started with NGINX](#getting-started-with-nginx) + - [Installing SSL certificates and enabling TLS encryption](#installing-ssl-certificates-and-enabling-tls-encryption) + - [Load Balancing](#load-balancing) + - [Rate limiting](#rate-limiting) + - [Content caching](#content-caching) +- [Building from source](#building-from-source) + - [Installing dependencies](#installing-dependencies) + - [Cloning the NGINX GitHub repository](#cloning-the-nginx-github-repository) + - [Configuring the build](#configuring-the-build) + - [Compiling](#compiling) + - [Location of binary and installation](#location-of-binary-and-installation) + - [Running and testing the installed binary](#running-and-testing-the-installed-binary) +- [Asking questions and reporting issues](#asking-questions-and-reporting-issues) +- [Contributing code](#contributing-code) +- [Additional help and resources](#additional-help-and-resources) +- [Changelog](#changelog) +- [License](#license) + +# How it works +NGINX is installed software with binary packages available for all major operating systems and Linux distributions. See [Tested OS and Platforms](https://nginx.org/en/#tested_os_and_platforms) for a full list of compatible systems. + +> [!IMPORTANT] +> While nearly all popular Linux-based operating systems are distributed with a community version of nginx, we highly advise installation and usage of official [packages](https://nginx.org/en/linux_packages.html) or sources from this repository. Doing so ensures that you're using the most recent release or source code, including the latest feature-set, fixes and security patches. + +## Modules +NGINX is comprised of individual modules, each extending core functionality by providing additional, configurable features. See "Modules reference" at the bottom of [nginx documentation](https://nginx.org/en/docs/) for a complete list of official modules. + +NGINX modules can be built and distributed as static or dynamic modules. Static modules are defined at build-time, compiled, and distributed in the resulting binaries. See [Dynamic Modules](#dynamic-modules) for more information on how they work, as well as, how to obtain, install, and configure them. + +> [!TIP] +> You can issue the following command to see which static modules your NGINX binaries were built with: +```bash +nginx -V +``` +> See [Configuring the build](#configuring-the-build) for information on how to include specific Static modules into your nginx build. + + +## Configurations +NGINX is highly flexible and configurable. Provisioning the software is achieved via text-based config file(s) accepting parameters called "[Directives](https://nginx.org/en/docs/dirindex.html)". See [Configuration File's Structure](https://nginx.org/en/docs/beginners_guide.html#conf_structure) for a comprehensive description of how NGINX configuration files work. + +> [!NOTE] +> The set of directives available to your distribution of NGINX is dependent on which [modules](#modules) have been made available to it. + +## Runtime +Rather than running in a single, monolithic process, NGINX is architected to scale beyond Operating System process limitations by operating as a collection of processes. They include: +- A "master" process that maintains worker processes, as well as, reads and evaluates configuration files. +- One or more "worker" processes that process data (eg. HTTP requests). + +The number of [worker processes](https://nginx.org/en/docs/ngx_core_module.html#worker_processes) is defined in the configuration file and may be fixed for a given configuration or automatically adjusted to the number of available CPU cores. In most cases, the latter option optimally balances load across available system resources, as NGINX is designed to efficiently distribute work across all worker processes. + +> [!TIP] +> Processes synchronize data through shared memory. For this reason, many NGINX directives require the allocation of shared memory zones. As an example, when configuring [rate limiting](https://nginx.org/en/docs/http/ngx_http_limit_req_module.html#limit_req), connecting clients may need to be tracked in a [common memory zone](https://nginx.org/en/docs/http/ngx_http_limit_req_module.html#limit_req_zone) so all worker processes can know how many times a particular client has accessed the server in a span of time. + +# Downloading and installing +Follow these steps to download and install precompiled NGINX binaries. You may also choose to [build NGINX locally from source code](#building-from-source). + +## Stable and Mainline binaries +NGINX binaries are built and distributed in two versions: stable and mainline. Stable binaries are built from stable branches and only contain critical fixes backported from the mainline version. Mainline binaries are built from the [master branch](https://github.com/nginx/nginx/tree/master) and contain the latest features and bugfixes. You'll need to [decide which is appropriate for your purposes](https://docs.nginx.com/nginx/admin-guide/installing-nginx/installing-nginx-open-source/#choosing-between-a-stable-or-a-mainline-version). + +## Linux binary installation process +The NGINX binary installation process takes advantage of package managers native to specific Linux distributions. For this reason, first-time installations involve adding the official NGINX package repository to your system's package manager. Follow [these steps](https://nginx.org/en/linux_packages.html) to download, verify, and install NGINX binaries using the package manager appropriate for your Linux distribution. + +### Upgrades +Future upgrades to the latest version can be managed using the same package manager without the need to manually download and verify binaries. + +## FreeBSD installation process +For more information on installing NGINX on FreeBSD system, visit https://nginx.org/en/docs/install.html + +## Windows executables +Windows executables for mainline and stable releases can be found on the main [NGINX download page](https://nginx.org/en/download.html). Note that the current implementation of NGINX for Windows is at the Proof-of-Concept stage and should only be used for development and testing purposes. For additional information, please see [nginx for Windows](https://nginx.org/en/docs/windows.html). + +## Dynamic modules +NGINX version 1.9.11 added support for [Dynamic Modules](https://nginx.org/en/docs/ngx_core_module.html#load_module). Unlike Static modules, dynamically built modules can be downloaded, installed, and configured after the core NGINX binaries have been built. [Official dynamic module binaries](https://nginx.org/en/linux_packages.html#dynmodules) are available from the same package repository as the core NGINX binaries described in previous steps. + +> [!TIP] +> [NGINX JavaScript (njs)](https://github.com/nginx/njs), is a popular NGINX dynamic module that enables the extension of core NGINX functionality using familiar JavaScript syntax. + +> [!IMPORTANT] +> If desired, dynamic modules can also be built statically into NGINX at compile time. + +# Getting started with NGINX +For a gentle introduction to NGINX basics, please see our [Beginner’s Guide](https://nginx.org/en/docs/beginners_guide.html). + +## Installing SSL certificates and enabling TLS encryption +See [Configuring HTTPS servers](https://nginx.org/en/docs/http/configuring_https_servers.html) for a quick guide on how to enable secure traffic to your NGINX installation. + +## Load Balancing +For a quick start guide on configuring NGINX as a Load Balancer, please see [Using nginx as HTTP load balancer](https://nginx.org/en/docs/http/load_balancing.html). + +## Rate limiting +See our [Rate Limiting with NGINX](https://blog.nginx.org/blog/rate-limiting-nginx) blog post for an overview of core concepts for provisioning NGINX as an API Gateway. + +## Content caching +See [A Guide to Caching with NGINX and NGINX Plus](https://blog.nginx.org/blog/nginx-caching-guide) blog post for an overview of how to use NGINX as a content cache (e.g. edge server of a content delivery network). + +# Building from source +The following steps can be used to build NGINX from source code available in this repository. + +## Installing dependencies +Most Linux distributions will require several dependencies to be installed in order to build NGINX. The following instructions are specific to the `apt` package manager, widely available on most Ubuntu/Debian distributions and their derivatives. + +> [!TIP] +> It is always a good idea to update your package repository lists prior to installing new packages. +> ```bash +> sudo apt update +> ``` + +### Installing compiler and make utility +Use the following command to install the GNU C compiler and Make utility. + +```bash +sudo apt install gcc make +``` + +### Installing dependency libraries + +```bash +sudo apt install libpcre3-dev zlib1g-dev +``` + +> [!WARNING] +> This is the minimal set of dependency libraries needed to build NGINX with rewriting and gzip capabilities. Other dependencies may be required if you choose to build NGINX with additional modules. Monitor the output of the `configure` command discussed in the following sections for information on which modules may be missing. For example, if you plan to use SSL certificates to encrypt traffic with TLS, you'll need to install the OpenSSL library. To do so, issue the following command. + +>```bash +>sudo apt install libssl-dev + +## Cloning the NGINX GitHub repository +Using your preferred method, clone the NGINX repository into your development directory. See [Cloning a GitHub Repository](https://docs.github.com/en/repositories/creating-and-managing-repositories/cloning-a-repository) for additional help. + +```bash +git clone https://github.com/nginx/nginx.git +``` + +## Configuring the build +Prior to building NGINX, you must run the `configure` script with [appropriate flags](https://nginx.org/en/docs/configure.html). This will generate a Makefile in your NGINX source root directory that can then be used to compile NGINX with [options specified during configuration](https://nginx.org/en/docs/configure.html). + +From the NGINX source code repository's root directory: + +```bash +auto/configure +``` + +> [!IMPORTANT] +> Configuring the build without any flags will compile NGINX with the default set of options. Please refer to https://nginx.org/en/docs/configure.html for a full list of available build configuration options. + +## Compiling +The `configure` script will generate a `Makefile` in the NGINX source root directory upon successful execution. To compile NGINX into a binary, issue the following command from that same directory: + +```bash +make +``` + +## Location of binary and installation +After successful compilation, a binary will be generated at `/objs/nginx`. To install this binary, issue the following command from the source root directory: + +```bash +sudo make install +``` + +> [!IMPORTANT] +> The binary will be installed into the `/usr/local/nginx/` directory. + +## Running and testing the installed binary +To run the installed binary, issue the following command: + +```bash +sudo /usr/local/nginx/sbin/nginx +``` + +You may test NGINX operation using `curl`. + +```bash +curl localhost +``` + +The output of which should start with: + +```html + + + +Welcome to nginx! +``` + +# Asking questions and reporting issues +We encourage you to engage with us. +- [NGINX GitHub Discussions](https://github.com/nginx/nginx/discussions), is the go-to place to start asking questions and sharing your thoughts. +- Our [GitHub Issues](https://github.com/nginx/nginx/issues) page offers space to submit and discuss specific issues, report bugs, and suggest enhancements. + +# Contributing code +Please see the [Contributing](CONTRIBUTING.md) guide for information on how to contribute code. + +# Additional help and resources +- See the [NGINX Community Blog](https://blog.nginx.org/) for more tips, tricks and HOW-TOs related to NGINX and related projects. +- Access [nginx.org](https://nginx.org/), your go-to source for all documentation, information and software related to the NGINX suite of projects. + +# Changelog +See our [changelog](https://nginx.org/en/CHANGES) to keep track of updates. + +# License +[2-clause BSD-like license](LICENSE) + +--- +Additional documentation available at: https://nginx.org/en/docs diff --git a/misc/GNUmakefile b/misc/GNUmakefile index 4b6a1275916..fcf34ec34a5 100644 --- a/misc/GNUmakefile +++ b/misc/GNUmakefile @@ -92,7 +92,7 @@ zip: export sed -i '' -e "s/$$/`printf '\r'`/" $(TEMP)/$(NGINX)/conf/* mv $(TEMP)/$(NGINX)/LICENSE $(TEMP)/$(NGINX)/docs.new - mv $(TEMP)/$(NGINX)/README $(TEMP)/$(NGINX)/docs.new + mv $(TEMP)/$(NGINX)/README.md $(TEMP)/$(NGINX)/docs.new mv $(TEMP)/$(NGINX)/docs/html $(TEMP)/$(NGINX) rm -r $(TEMP)/$(NGINX)/docs From 51857ce40400b48bc8900b9e3930cf7474fa0c41 Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Mon, 9 Sep 2024 23:22:34 +0800 Subject: [PATCH 167/279] Fixed a typo of bpf makefile debug option. --- src/event/quic/bpf/makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/event/quic/bpf/makefile b/src/event/quic/bpf/makefile index b4d758f33bc..277aa315166 100644 --- a/src/event/quic/bpf/makefile +++ b/src/event/quic/bpf/makefile @@ -25,6 +25,6 @@ clean: @rm -f $(RESULT) *.o debug: $(PROGNAME).o - llvm-objdump -S -no-show-raw-insn $< + llvm-objdump -S --no-show-raw-insn $< .DELETE_ON_ERROR: From f36ff3550a7271a618edb119f064dddd086cc380 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Mon, 9 Sep 2024 19:02:27 +0400 Subject: [PATCH 168/279] SSL: moved certificate storage out of exdata. Instead of cross-linking the objects using exdata, pointers to configured certificates are now stored in ngx_ssl_t, and OCSP staples are now accessed with rbtree in it. This allows sharing these objects between SSL contexts. Based on previous work by Mini Hawthorne. --- src/event/ngx_event_openssl.c | 78 ++++++++++---------------- src/event/ngx_event_openssl.h | 9 ++- src/event/ngx_event_openssl_stapling.c | 68 +++++++++++++++++----- 3 files changed, 89 insertions(+), 66 deletions(-) diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c index 8d1f5695cbd..3ed0030622c 100644 --- a/src/event/ngx_event_openssl.c +++ b/src/event/ngx_event_openssl.c @@ -131,10 +131,8 @@ int ngx_ssl_server_conf_index; int ngx_ssl_session_cache_index; int ngx_ssl_ticket_keys_index; int ngx_ssl_ocsp_index; -int ngx_ssl_certificate_index; -int ngx_ssl_next_certificate_index; +int ngx_ssl_index; int ngx_ssl_certificate_name_index; -int ngx_ssl_stapling_index; ngx_int_t @@ -258,21 +256,14 @@ ngx_ssl_init(ngx_log_t *log) return NGX_ERROR; } - ngx_ssl_certificate_index = SSL_CTX_get_ex_new_index(0, NULL, NULL, NULL, - NULL); - if (ngx_ssl_certificate_index == -1) { + ngx_ssl_index = SSL_CTX_get_ex_new_index(0, NULL, NULL, NULL, NULL); + + if (ngx_ssl_index == -1) { ngx_ssl_error(NGX_LOG_ALERT, log, 0, "SSL_CTX_get_ex_new_index() failed"); return NGX_ERROR; } - ngx_ssl_next_certificate_index = X509_get_ex_new_index(0, NULL, NULL, NULL, - NULL); - if (ngx_ssl_next_certificate_index == -1) { - ngx_ssl_error(NGX_LOG_ALERT, log, 0, "X509_get_ex_new_index() failed"); - return NGX_ERROR; - } - ngx_ssl_certificate_name_index = X509_get_ex_new_index(0, NULL, NULL, NULL, NULL); @@ -281,13 +272,6 @@ ngx_ssl_init(ngx_log_t *log) return NGX_ERROR; } - ngx_ssl_stapling_index = X509_get_ex_new_index(0, NULL, NULL, NULL, NULL); - - if (ngx_ssl_stapling_index == -1) { - ngx_ssl_error(NGX_LOG_ALERT, log, 0, "X509_get_ex_new_index() failed"); - return NGX_ERROR; - } - return NGX_OK; } @@ -308,12 +292,15 @@ ngx_ssl_create(ngx_ssl_t *ssl, ngx_uint_t protocols, void *data) return NGX_ERROR; } - if (SSL_CTX_set_ex_data(ssl->ctx, ngx_ssl_certificate_index, NULL) == 0) { + if (SSL_CTX_set_ex_data(ssl->ctx, ngx_ssl_index, ssl) == 0) { ngx_ssl_error(NGX_LOG_EMERG, ssl->log, 0, "SSL_CTX_set_ex_data() failed"); return NGX_ERROR; } + ngx_rbtree_init(&ssl->staple_rbtree, &ssl->staple_sentinel, + ngx_rbtree_insert_value); + ssl->buffer_size = NGX_SSL_BUFSIZE; /* client side options */ @@ -458,7 +445,7 @@ ngx_ssl_certificate(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *cert, ngx_str_t *key, ngx_array_t *passwords) { char *err; - X509 *x509; + X509 *x509, **elm; EVP_PKEY *pkey; STACK_OF(X509) *chain; @@ -490,29 +477,29 @@ ngx_ssl_certificate(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *cert, return NGX_ERROR; } - if (X509_set_ex_data(x509, ngx_ssl_next_certificate_index, - SSL_CTX_get_ex_data(ssl->ctx, ngx_ssl_certificate_index)) - == 0) - { - ngx_ssl_error(NGX_LOG_EMERG, ssl->log, 0, "X509_set_ex_data() failed"); - X509_free(x509); - sk_X509_pop_free(chain, X509_free); - return NGX_ERROR; + if (ssl->certs.elts == NULL) { + if (ngx_array_init(&ssl->certs, cf->pool, 1, sizeof(X509 *)) + != NGX_OK) + { + X509_free(x509); + sk_X509_pop_free(chain, X509_free); + return NGX_ERROR; + } } - if (SSL_CTX_set_ex_data(ssl->ctx, ngx_ssl_certificate_index, x509) == 0) { - ngx_ssl_error(NGX_LOG_EMERG, ssl->log, 0, - "SSL_CTX_set_ex_data() failed"); + elm = ngx_array_push(&ssl->certs); + if (elm == NULL) { X509_free(x509); sk_X509_pop_free(chain, X509_free); return NGX_ERROR; } + *elm = x509; + /* * Note that x509 is not freed here, but will be instead freed in * ngx_ssl_cleanup_ctx(). This is because we need to preserve all - * certificates to be able to iterate all of them through exdata - * (ngx_ssl_certificate_index, ngx_ssl_next_certificate_index), + * certificates to be able to iterate all of them through ssl->certs, * while OpenSSL can free a certificate if it is replaced with another * certificate of the same type. */ @@ -3820,10 +3807,9 @@ ngx_ssl_session_id_context(ngx_ssl_t *ssl, ngx_str_t *sess_ctx, goto failed; } - for (cert = SSL_CTX_get_ex_data(ssl->ctx, ngx_ssl_certificate_index); - cert; - cert = X509_get_ex_data(cert, ngx_ssl_next_certificate_index)) - { + for (k = 0; k < ssl->certs.nelts; k++) { + cert = ((X509 **) ssl->certs.elts)[k]; + if (X509_digest(cert, EVP_sha1(), buf, &len) == 0) { ngx_ssl_error(NGX_LOG_EMERG, ssl->log, 0, "X509_digest() failed"); @@ -3837,9 +3823,7 @@ ngx_ssl_session_id_context(ngx_ssl_t *ssl, ngx_str_t *sess_ctx, } } - if (SSL_CTX_get_ex_data(ssl->ctx, ngx_ssl_certificate_index) == NULL - && certificates != NULL) - { + if (ssl->certs.nelts == 0 && certificates != NULL) { /* * If certificates are loaded dynamically, we use certificate * names as specified in the configuration (with variables). @@ -4851,14 +4835,12 @@ ngx_ssl_cleanup_ctx(void *data) { ngx_ssl_t *ssl = data; - X509 *cert, *next; - - cert = SSL_CTX_get_ex_data(ssl->ctx, ngx_ssl_certificate_index); + X509 *cert; + ngx_uint_t i; - while (cert) { - next = X509_get_ex_data(cert, ngx_ssl_next_certificate_index); + for (i = 0; i < ssl->certs.nelts; i++) { + cert = ((X509 **) ssl->certs.elts)[i]; X509_free(cert); - cert = next; } SSL_CTX_free(ssl->ctx); diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h index ebb2c35bf28..41566422a8a 100644 --- a/src/event/ngx_event_openssl.h +++ b/src/event/ngx_event_openssl.h @@ -90,6 +90,11 @@ struct ngx_ssl_s { SSL_CTX *ctx; ngx_log_t *log; size_t buffer_size; + + ngx_array_t certs; + + ngx_rbtree_t staple_rbtree; + ngx_rbtree_node_t staple_sentinel; }; @@ -330,10 +335,8 @@ extern int ngx_ssl_server_conf_index; extern int ngx_ssl_session_cache_index; extern int ngx_ssl_ticket_keys_index; extern int ngx_ssl_ocsp_index; -extern int ngx_ssl_certificate_index; -extern int ngx_ssl_next_certificate_index; +extern int ngx_ssl_index; extern int ngx_ssl_certificate_name_index; -extern int ngx_ssl_stapling_index; #endif /* _NGX_EVENT_OPENSSL_H_INCLUDED_ */ diff --git a/src/event/ngx_event_openssl_stapling.c b/src/event/ngx_event_openssl_stapling.c index e9bb8354eee..a0a8031c767 100644 --- a/src/event/ngx_event_openssl_stapling.c +++ b/src/event/ngx_event_openssl_stapling.c @@ -15,6 +15,8 @@ typedef struct { + ngx_rbtree_node_t node; + ngx_str_t staple; ngx_msec_t timeout; @@ -154,6 +156,7 @@ static ngx_int_t ngx_ssl_stapling_responder(ngx_conf_t *cf, ngx_ssl_t *ssl, static int ngx_ssl_certificate_status_callback(ngx_ssl_conn_t *ssl_conn, void *data); +static ngx_ssl_stapling_t *ngx_ssl_stapling_lookup(ngx_ssl_t *ssl, X509 *cert); static void ngx_ssl_stapling_update(ngx_ssl_stapling_t *staple); static void ngx_ssl_stapling_ocsp_handler(ngx_ssl_ocsp_ctx_t *ctx); @@ -195,12 +198,12 @@ ngx_int_t ngx_ssl_stapling(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *file, ngx_str_t *responder, ngx_uint_t verify) { - X509 *cert; + X509 *cert; + ngx_uint_t k; + + for (k = 0; k < ssl->certs.nelts; k++) { + cert = ((X509 **) ssl->certs.elts)[k]; - for (cert = SSL_CTX_get_ex_data(ssl->ctx, ngx_ssl_certificate_index); - cert; - cert = X509_get_ex_data(cert, ngx_ssl_next_certificate_index)) - { if (ngx_ssl_stapling_certificate(cf, ssl, cert, file, responder, verify) != NGX_OK) { @@ -235,10 +238,9 @@ ngx_ssl_stapling_certificate(ngx_conf_t *cf, ngx_ssl_t *ssl, X509 *cert, cln->handler = ngx_ssl_stapling_cleanup; cln->data = staple; - if (X509_set_ex_data(cert, ngx_ssl_stapling_index, staple) == 0) { - ngx_ssl_error(NGX_LOG_EMERG, ssl->log, 0, "X509_set_ex_data() failed"); - return NGX_ERROR; - } + staple->node.key = (ngx_rbtree_key_t) cert; + + ngx_rbtree_insert(&ssl->staple_rbtree, &staple->node); #ifdef SSL_CTRL_SELECT_CURRENT_CERT /* OpenSSL 1.0.2+ */ @@ -545,14 +547,21 @@ ngx_int_t ngx_ssl_stapling_resolver(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_resolver_t *resolver, ngx_msec_t resolver_timeout) { - X509 *cert; + ngx_rbtree_t *tree; + ngx_rbtree_node_t *node; ngx_ssl_stapling_t *staple; - for (cert = SSL_CTX_get_ex_data(ssl->ctx, ngx_ssl_certificate_index); - cert; - cert = X509_get_ex_data(cert, ngx_ssl_next_certificate_index)) + tree = &ssl->staple_rbtree; + + if (tree->root == tree->sentinel) { + return NGX_OK; + } + + for (node = ngx_rbtree_min(tree->root, tree->sentinel); + node; + node = ngx_rbtree_next(tree, node)) { - staple = X509_get_ex_data(cert, ngx_ssl_stapling_index); + staple = ngx_rbtree_data(node, ngx_ssl_stapling_t, node); staple->resolver = resolver; staple->resolver_timeout = resolver_timeout; } @@ -567,6 +576,8 @@ ngx_ssl_certificate_status_callback(ngx_ssl_conn_t *ssl_conn, void *data) int rc; X509 *cert; u_char *p; + SSL_CTX *ssl_ctx; + ngx_ssl_t *ssl; ngx_connection_t *c; ngx_ssl_stapling_t *staple; @@ -583,7 +594,10 @@ ngx_ssl_certificate_status_callback(ngx_ssl_conn_t *ssl_conn, void *data) return rc; } - staple = X509_get_ex_data(cert, ngx_ssl_stapling_index); + ssl_ctx = SSL_get_SSL_CTX(ssl_conn); + ssl = SSL_CTX_get_ex_data(ssl_ctx, ngx_ssl_index); + + staple = ngx_ssl_stapling_lookup(ssl, cert); if (staple == NULL) { return rc; @@ -613,6 +627,30 @@ ngx_ssl_certificate_status_callback(ngx_ssl_conn_t *ssl_conn, void *data) } +static ngx_ssl_stapling_t * +ngx_ssl_stapling_lookup(ngx_ssl_t *ssl, X509 *cert) +{ + ngx_rbtree_key_t key; + ngx_rbtree_node_t *node, *sentinel; + + node = ssl->staple_rbtree.root; + sentinel = ssl->staple_rbtree.sentinel; + key = (ngx_rbtree_key_t) cert; + + while (node != sentinel) { + + if (key != node->key) { + node = (key < node->key) ? node->left : node->right; + continue; + } + + return ngx_rbtree_data(node, ngx_ssl_stapling_t, node); + } + + return NULL; +} + + static void ngx_ssl_stapling_update(ngx_ssl_stapling_t *staple) { From 7d7e8d2cb8d16e409e0d4c777b30f1d8d7838c7b Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Mon, 9 Sep 2024 19:03:20 +0400 Subject: [PATCH 169/279] SSL: object caching. Added ngx_openssl_cache_module, which indexes a type-aware object cache. It maps an id to a unique instance, and provides references to it, which are dropped when the cycle's pool is destroyed. The cache will be used in subsequent patches. Based on previous work by Mini Hawthorne. --- auto/modules | 3 +- src/event/ngx_event_openssl.h | 5 + src/event/ngx_event_openssl_cache.c | 311 ++++++++++++++++++++++++++++ 3 files changed, 318 insertions(+), 1 deletion(-) create mode 100644 src/event/ngx_event_openssl_cache.c diff --git a/auto/modules b/auto/modules index 1a5e4212d0b..38b3aba78b5 100644 --- a/auto/modules +++ b/auto/modules @@ -1307,10 +1307,11 @@ fi if [ $USE_OPENSSL = YES ]; then ngx_module_type=CORE - ngx_module_name=ngx_openssl_module + ngx_module_name="ngx_openssl_module ngx_openssl_cache_module" ngx_module_incs= ngx_module_deps=src/event/ngx_event_openssl.h ngx_module_srcs="src/event/ngx_event_openssl.c + src/event/ngx_event_openssl_cache.c src/event/ngx_event_openssl_stapling.c" ngx_module_libs= ngx_module_link=YES diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h index 41566422a8a..fa2282ce83a 100644 --- a/src/event/ngx_event_openssl.h +++ b/src/event/ngx_event_openssl.h @@ -224,6 +224,11 @@ ngx_int_t ngx_ssl_ocsp_get_status(ngx_connection_t *c, const char **s); void ngx_ssl_ocsp_cleanup(ngx_connection_t *c); ngx_int_t ngx_ssl_ocsp_cache_init(ngx_shm_zone_t *shm_zone, void *data); +void *ngx_ssl_cache_fetch(ngx_conf_t *cf, ngx_uint_t index, char **err, + ngx_str_t *path, void *data); +void *ngx_ssl_cache_connection_fetch(ngx_pool_t *pool, ngx_uint_t index, + char **err, ngx_str_t *path, void *data); + ngx_array_t *ngx_ssl_read_password_file(ngx_conf_t *cf, ngx_str_t *file); ngx_array_t *ngx_ssl_preserve_passwords(ngx_conf_t *cf, ngx_array_t *passwords); diff --git a/src/event/ngx_event_openssl_cache.c b/src/event/ngx_event_openssl_cache.c new file mode 100644 index 00000000000..98867de5063 --- /dev/null +++ b/src/event/ngx_event_openssl_cache.c @@ -0,0 +1,311 @@ + +/* + * Copyright (C) Nginx, Inc. + */ + + +#include +#include +#include + + +#define NGX_SSL_CACHE_PATH 0 + + +typedef struct { + unsigned type:2; + unsigned len:30; + u_char *data; +} ngx_ssl_cache_key_t; + + +typedef void *(*ngx_ssl_cache_create_pt)(ngx_ssl_cache_key_t *id, char **err, + void *data); +typedef void (*ngx_ssl_cache_free_pt)(void *data); +typedef void *(*ngx_ssl_cache_ref_pt)(char **err, void *data); + + +typedef struct { + ngx_ssl_cache_create_pt create; + ngx_ssl_cache_free_pt free; + ngx_ssl_cache_ref_pt ref; +} ngx_ssl_cache_type_t; + + +typedef struct { + ngx_rbtree_node_t node; + ngx_ssl_cache_key_t id; + ngx_ssl_cache_type_t *type; + void *value; +} ngx_ssl_cache_node_t; + + +typedef struct { + ngx_rbtree_t rbtree; + ngx_rbtree_node_t sentinel; +} ngx_ssl_cache_t; + + +static ngx_int_t ngx_ssl_cache_init_key(ngx_pool_t *pool, ngx_uint_t index, + ngx_str_t *path, ngx_ssl_cache_key_t *id); +static ngx_ssl_cache_node_t *ngx_ssl_cache_lookup(ngx_ssl_cache_t *cache, + ngx_ssl_cache_type_t *type, ngx_ssl_cache_key_t *id, uint32_t hash); + +static void *ngx_openssl_cache_create_conf(ngx_cycle_t *cycle); +static void ngx_ssl_cache_cleanup(void *data); +static void ngx_ssl_cache_node_insert(ngx_rbtree_node_t *temp, + ngx_rbtree_node_t *node, ngx_rbtree_node_t *sentinel); + + +static ngx_core_module_t ngx_openssl_cache_module_ctx = { + ngx_string("openssl_cache"), + ngx_openssl_cache_create_conf, + NULL +}; + + +ngx_module_t ngx_openssl_cache_module = { + NGX_MODULE_V1, + &ngx_openssl_cache_module_ctx, /* module context */ + NULL, /* module directives */ + NGX_CORE_MODULE, /* module type */ + NULL, /* init master */ + NULL, /* init module */ + NULL, /* init process */ + NULL, /* init thread */ + NULL, /* exit thread */ + NULL, /* exit process */ + NULL, /* exit master */ + NGX_MODULE_V1_PADDING +}; + + +static ngx_ssl_cache_type_t ngx_ssl_cache_types[] = { + +}; + + +void * +ngx_ssl_cache_fetch(ngx_conf_t *cf, ngx_uint_t index, char **err, + ngx_str_t *path, void *data) +{ + uint32_t hash; + ngx_ssl_cache_t *cache; + ngx_ssl_cache_key_t id; + ngx_ssl_cache_type_t *type; + ngx_ssl_cache_node_t *cn; + + if (ngx_ssl_cache_init_key(cf->pool, index, path, &id) != NGX_OK) { + return NULL; + } + + cache = (ngx_ssl_cache_t *) ngx_get_conf(cf->cycle->conf_ctx, + ngx_openssl_cache_module); + + type = &ngx_ssl_cache_types[index]; + hash = ngx_murmur_hash2(id.data, id.len); + + cn = ngx_ssl_cache_lookup(cache, type, &id, hash); + if (cn != NULL) { + return type->ref(err, cn->value); + } + + cn = ngx_palloc(cf->pool, sizeof(ngx_ssl_cache_node_t) + id.len + 1); + if (cn == NULL) { + return NULL; + } + + cn->node.key = hash; + cn->id.data = (u_char *)(cn + 1); + cn->id.len = id.len; + cn->id.type = id.type; + cn->type = type; + + ngx_cpystrn(cn->id.data, id.data, id.len + 1); + + cn->value = type->create(&id, err, data); + if (cn->value == NULL) { + return NULL; + } + + ngx_rbtree_insert(&cache->rbtree, &cn->node); + + return type->ref(err, cn->value); +} + + +void * +ngx_ssl_cache_connection_fetch(ngx_pool_t *pool, ngx_uint_t index, char **err, + ngx_str_t *path, void *data) +{ + ngx_ssl_cache_key_t id; + + if (ngx_ssl_cache_init_key(pool, index, path, &id) != NGX_OK) { + return NULL; + } + + return ngx_ssl_cache_types[index].create(&id, err, data); +} + + +static ngx_int_t +ngx_ssl_cache_init_key(ngx_pool_t *pool, ngx_uint_t index, ngx_str_t *path, + ngx_ssl_cache_key_t *id) +{ + if (ngx_get_full_name(pool, (ngx_str_t *) &ngx_cycle->conf_prefix, path) + != NGX_OK) + { + return NGX_ERROR; + } + + id->type = NGX_SSL_CACHE_PATH; + + id->len = path->len; + id->data = path->data; + + return NGX_OK; +} + + +static ngx_ssl_cache_node_t * +ngx_ssl_cache_lookup(ngx_ssl_cache_t *cache, ngx_ssl_cache_type_t *type, + ngx_ssl_cache_key_t *id, uint32_t hash) +{ + ngx_int_t rc; + ngx_rbtree_node_t *node, *sentinel; + ngx_ssl_cache_node_t *cn; + + node = cache->rbtree.root; + sentinel = cache->rbtree.sentinel; + + while (node != sentinel) { + + if (hash < node->key) { + node = node->left; + continue; + } + + if (hash > node->key) { + node = node->right; + continue; + } + + /* hash == node->key */ + + cn = (ngx_ssl_cache_node_t *) node; + + if (type < cn->type) { + node = node->left; + continue; + } + + if (type > cn->type) { + node = node->right; + continue; + } + + /* type == cn->type */ + + rc = ngx_memn2cmp(id->data, cn->id.data, id->len, cn->id.len); + + if (rc == 0) { + return cn; + } + + node = (rc < 0) ? node->left : node->right; + } + + return NULL; +} + + +static void * +ngx_openssl_cache_create_conf(ngx_cycle_t *cycle) +{ + ngx_ssl_cache_t *cache; + ngx_pool_cleanup_t *cln; + + cache = ngx_pcalloc(cycle->pool, sizeof(ngx_ssl_cache_t)); + if (cache == NULL) { + return NULL; + } + + cln = ngx_pool_cleanup_add(cycle->pool, 0); + if (cln == NULL) { + return NULL; + } + + cln->handler = ngx_ssl_cache_cleanup; + cln->data = cache; + + ngx_rbtree_init(&cache->rbtree, &cache->sentinel, + ngx_ssl_cache_node_insert); + + return cache; +} + + +static void +ngx_ssl_cache_cleanup(void *data) +{ + ngx_ssl_cache_t *cache = data; + + ngx_rbtree_t *tree; + ngx_rbtree_node_t *node; + ngx_ssl_cache_node_t *cn; + + tree = &cache->rbtree; + + if (tree->root == tree->sentinel) { + return; + } + + for (node = ngx_rbtree_min(tree->root, tree->sentinel); + node; + node = ngx_rbtree_next(tree, node)) + { + cn = ngx_rbtree_data(node, ngx_ssl_cache_node_t, node); + cn->type->free(cn->value); + } +} + + +static void +ngx_ssl_cache_node_insert(ngx_rbtree_node_t *temp, + ngx_rbtree_node_t *node, ngx_rbtree_node_t *sentinel) +{ + ngx_rbtree_node_t **p; + ngx_ssl_cache_node_t *n, *t; + + for ( ;; ) { + + n = ngx_rbtree_data(node, ngx_ssl_cache_node_t, node); + t = ngx_rbtree_data(temp, ngx_ssl_cache_node_t, node); + + if (node->key != temp->key) { + + p = (node->key < temp->key) ? &temp->left : &temp->right; + + } else if (n->type != t->type) { + + p = (n->type < t->type) ? &temp->left : &temp->right; + + } else { + + p = (ngx_memn2cmp(n->id.data, t->id.data, n->id.len, t->id.len) + < 0) ? &temp->left : &temp->right; + } + + if (*p == sentinel) { + break; + } + + temp = *p; + } + + *p = node; + node->parent = temp; + node->left = sentinel; + node->right = sentinel; + ngx_rbt_red(node); +} From 78ed123e71aab17ffecfa0b2b27a349cfb4b2502 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Mon, 9 Sep 2024 19:03:52 +0400 Subject: [PATCH 170/279] SSL: caching certificates. Certificate chains are now loaded once. The certificate cache provides each chain as a unique stack of reference counted elements. This shallow copy is required because OpenSSL stacks aren't reference counted. Based on previous work by Mini Hawthorne. --- src/event/ngx_event_openssl.c | 105 ++--------------- src/event/ngx_event_openssl.h | 3 + src/event/ngx_event_openssl_cache.c | 167 +++++++++++++++++++++++++++- 3 files changed, 173 insertions(+), 102 deletions(-) diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c index 3ed0030622c..018d0301627 100644 --- a/src/event/ngx_event_openssl.c +++ b/src/event/ngx_event_openssl.c @@ -18,8 +18,6 @@ typedef struct { } ngx_openssl_conf_t; -static X509 *ngx_ssl_load_certificate(ngx_pool_t *pool, char **err, - ngx_str_t *cert, STACK_OF(X509) **chain); static EVP_PKEY *ngx_ssl_load_certificate_key(ngx_pool_t *pool, char **err, ngx_str_t *key, ngx_array_t *passwords); static int ngx_ssl_password_callback(char *buf, int size, int rwflag, @@ -449,8 +447,8 @@ ngx_ssl_certificate(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *cert, EVP_PKEY *pkey; STACK_OF(X509) *chain; - x509 = ngx_ssl_load_certificate(cf->pool, &err, cert, &chain); - if (x509 == NULL) { + chain = ngx_ssl_cache_fetch(cf, NGX_SSL_CACHE_CERT, &err, cert, NULL); + if (chain == NULL) { if (err != NULL) { ngx_ssl_error(NGX_LOG_EMERG, ssl->log, 0, "cannot load certificate \"%s\": %s", @@ -460,6 +458,8 @@ ngx_ssl_certificate(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *cert, return NGX_ERROR; } + x509 = sk_X509_shift(chain); + if (SSL_CTX_use_certificate(ssl->ctx, x509) == 0) { ngx_ssl_error(NGX_LOG_EMERG, ssl->log, 0, "SSL_CTX_use_certificate(\"%s\") failed", cert->data); @@ -570,8 +570,9 @@ ngx_ssl_connection_certificate(ngx_connection_t *c, ngx_pool_t *pool, EVP_PKEY *pkey; STACK_OF(X509) *chain; - x509 = ngx_ssl_load_certificate(pool, &err, cert, &chain); - if (x509 == NULL) { + chain = ngx_ssl_cache_connection_fetch(pool, NGX_SSL_CACHE_CERT, &err, + cert, NULL); + if (chain == NULL) { if (err != NULL) { ngx_ssl_error(NGX_LOG_ERR, c->log, 0, "cannot load certificate \"%s\": %s", @@ -581,6 +582,8 @@ ngx_ssl_connection_certificate(ngx_connection_t *c, ngx_pool_t *pool, return NGX_ERROR; } + x509 = sk_X509_shift(chain); + if (SSL_use_certificate(c->ssl->connection, x509) == 0) { ngx_ssl_error(NGX_LOG_ERR, c->log, 0, "SSL_use_certificate(\"%s\") failed", cert->data); @@ -632,96 +635,6 @@ ngx_ssl_connection_certificate(ngx_connection_t *c, ngx_pool_t *pool, } -static X509 * -ngx_ssl_load_certificate(ngx_pool_t *pool, char **err, ngx_str_t *cert, - STACK_OF(X509) **chain) -{ - BIO *bio; - X509 *x509, *temp; - u_long n; - - if (ngx_strncmp(cert->data, "data:", sizeof("data:") - 1) == 0) { - - bio = BIO_new_mem_buf(cert->data + sizeof("data:") - 1, - cert->len - (sizeof("data:") - 1)); - if (bio == NULL) { - *err = "BIO_new_mem_buf() failed"; - return NULL; - } - - } else { - - if (ngx_get_full_name(pool, (ngx_str_t *) &ngx_cycle->conf_prefix, cert) - != NGX_OK) - { - *err = NULL; - return NULL; - } - - bio = BIO_new_file((char *) cert->data, "r"); - if (bio == NULL) { - *err = "BIO_new_file() failed"; - return NULL; - } - } - - /* certificate itself */ - - x509 = PEM_read_bio_X509_AUX(bio, NULL, NULL, NULL); - if (x509 == NULL) { - *err = "PEM_read_bio_X509_AUX() failed"; - BIO_free(bio); - return NULL; - } - - /* rest of the chain */ - - *chain = sk_X509_new_null(); - if (*chain == NULL) { - *err = "sk_X509_new_null() failed"; - BIO_free(bio); - X509_free(x509); - return NULL; - } - - for ( ;; ) { - - temp = PEM_read_bio_X509(bio, NULL, NULL, NULL); - if (temp == NULL) { - n = ERR_peek_last_error(); - - if (ERR_GET_LIB(n) == ERR_LIB_PEM - && ERR_GET_REASON(n) == PEM_R_NO_START_LINE) - { - /* end of file */ - ERR_clear_error(); - break; - } - - /* some real error */ - - *err = "PEM_read_bio_X509() failed"; - BIO_free(bio); - X509_free(x509); - sk_X509_pop_free(*chain, X509_free); - return NULL; - } - - if (sk_X509_push(*chain, temp) == 0) { - *err = "sk_X509_push() failed"; - BIO_free(bio); - X509_free(x509); - sk_X509_pop_free(*chain, X509_free); - return NULL; - } - } - - BIO_free(bio); - - return x509; -} - - static EVP_PKEY * ngx_ssl_load_certificate_key(ngx_pool_t *pool, char **err, ngx_str_t *key, ngx_array_t *passwords) diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h index fa2282ce83a..addc7b4d3aa 100644 --- a/src/event/ngx_event_openssl.h +++ b/src/event/ngx_event_openssl.h @@ -193,6 +193,9 @@ typedef struct { #define NGX_SSL_BUFSIZE 16384 +#define NGX_SSL_CACHE_CERT 0 + + ngx_int_t ngx_ssl_init(ngx_log_t *log); ngx_int_t ngx_ssl_create(ngx_ssl_t *ssl, ngx_uint_t protocols, void *data); diff --git a/src/event/ngx_event_openssl_cache.c b/src/event/ngx_event_openssl_cache.c index 98867de5063..20b87b79a86 100644 --- a/src/event/ngx_event_openssl_cache.c +++ b/src/event/ngx_event_openssl_cache.c @@ -10,6 +10,7 @@ #define NGX_SSL_CACHE_PATH 0 +#define NGX_SSL_CACHE_DATA 1 typedef struct { @@ -51,6 +52,13 @@ static ngx_int_t ngx_ssl_cache_init_key(ngx_pool_t *pool, ngx_uint_t index, static ngx_ssl_cache_node_t *ngx_ssl_cache_lookup(ngx_ssl_cache_t *cache, ngx_ssl_cache_type_t *type, ngx_ssl_cache_key_t *id, uint32_t hash); +static void *ngx_ssl_cache_cert_create(ngx_ssl_cache_key_t *id, char **err, + void *data); +static void ngx_ssl_cache_cert_free(void *data); +static void *ngx_ssl_cache_cert_ref(char **err, void *data); + +static BIO *ngx_ssl_cache_create_bio(ngx_ssl_cache_key_t *id, char **err); + static void *ngx_openssl_cache_create_conf(ngx_cycle_t *cycle); static void ngx_ssl_cache_cleanup(void *data); static void ngx_ssl_cache_node_insert(ngx_rbtree_node_t *temp, @@ -82,6 +90,10 @@ ngx_module_t ngx_openssl_cache_module = { static ngx_ssl_cache_type_t ngx_ssl_cache_types[] = { + /* NGX_SSL_CACHE_CERT */ + { ngx_ssl_cache_cert_create, + ngx_ssl_cache_cert_free, + ngx_ssl_cache_cert_ref }, }; @@ -152,13 +164,18 @@ static ngx_int_t ngx_ssl_cache_init_key(ngx_pool_t *pool, ngx_uint_t index, ngx_str_t *path, ngx_ssl_cache_key_t *id) { - if (ngx_get_full_name(pool, (ngx_str_t *) &ngx_cycle->conf_prefix, path) - != NGX_OK) - { - return NGX_ERROR; - } + if (ngx_strncmp(path->data, "data:", sizeof("data:") - 1) == 0) { + id->type = NGX_SSL_CACHE_DATA; + + } else { + if (ngx_get_full_name(pool, (ngx_str_t *) &ngx_cycle->conf_prefix, path) + != NGX_OK) + { + return NGX_ERROR; + } - id->type = NGX_SSL_CACHE_PATH; + id->type = NGX_SSL_CACHE_PATH; + } id->len = path->len; id->data = path->data; @@ -219,6 +236,144 @@ ngx_ssl_cache_lookup(ngx_ssl_cache_t *cache, ngx_ssl_cache_type_t *type, } +static void * +ngx_ssl_cache_cert_create(ngx_ssl_cache_key_t *id, char **err, void *data) +{ + BIO *bio; + X509 *x509; + u_long n; + STACK_OF(X509) *chain; + + chain = sk_X509_new_null(); + if (chain == NULL) { + *err = "sk_X509_new_null() failed"; + return NULL; + } + + bio = ngx_ssl_cache_create_bio(id, err); + if (bio == NULL) { + sk_X509_pop_free(chain, X509_free); + return NULL; + } + + /* certificate itself */ + + x509 = PEM_read_bio_X509_AUX(bio, NULL, NULL, NULL); + if (x509 == NULL) { + *err = "PEM_read_bio_X509_AUX() failed"; + BIO_free(bio); + sk_X509_pop_free(chain, X509_free); + return NULL; + } + + if (sk_X509_push(chain, x509) == 0) { + *err = "sk_X509_push() failed"; + BIO_free(bio); + X509_free(x509); + sk_X509_pop_free(chain, X509_free); + return NULL; + } + + /* rest of the chain */ + + for ( ;; ) { + + x509 = PEM_read_bio_X509(bio, NULL, NULL, NULL); + if (x509 == NULL) { + n = ERR_peek_last_error(); + + if (ERR_GET_LIB(n) == ERR_LIB_PEM + && ERR_GET_REASON(n) == PEM_R_NO_START_LINE) + { + /* end of file */ + ERR_clear_error(); + break; + } + + /* some real error */ + + *err = "PEM_read_bio_X509() failed"; + BIO_free(bio); + sk_X509_pop_free(chain, X509_free); + return NULL; + } + + if (sk_X509_push(chain, x509) == 0) { + *err = "sk_X509_push() failed"; + BIO_free(bio); + X509_free(x509); + sk_X509_pop_free(chain, X509_free); + return NULL; + } + } + + BIO_free(bio); + + return chain; +} + + +static void +ngx_ssl_cache_cert_free(void *data) +{ + sk_X509_pop_free(data, X509_free); +} + + +static void * +ngx_ssl_cache_cert_ref(char **err, void *data) +{ + int n, i; + X509 *x509; + STACK_OF(X509) *chain; + + chain = sk_X509_dup(data); + if (chain == NULL) { + *err = "sk_X509_dup() failed"; + return NULL; + } + + n = sk_X509_num(chain); + + for (i = 0; i < n; i++) { + x509 = sk_X509_value(chain, i); + +#if (OPENSSL_VERSION_NUMBER >= 0x10100000L) + X509_up_ref(x509); +#else + CRYPTO_add(&x509->references, 1, CRYPTO_LOCK_X509); +#endif + } + + return chain; +} + + +static BIO * +ngx_ssl_cache_create_bio(ngx_ssl_cache_key_t *id, char **err) +{ + BIO *bio; + + if (id->type == NGX_SSL_CACHE_DATA) { + + bio = BIO_new_mem_buf(id->data + sizeof("data:") - 1, + id->len - (sizeof("data:") - 1)); + if (bio == NULL) { + *err = "BIO_new_mem_buf() failed"; + } + + return bio; + } + + bio = BIO_new_file((char *) id->data, "r"); + if (bio == NULL) { + *err = "BIO_new_file() failed"; + } + + return bio; +} + + static void * ngx_openssl_cache_create_conf(ngx_cycle_t *cycle) { From 7ea2fb6cb197925c7c0e35def9ece12d11b09bb9 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Mon, 9 Sep 2024 19:04:18 +0400 Subject: [PATCH 171/279] SSL: caching certificate keys. EVP_KEY objects are a reference-counted container for key material, shallow copies and OpenSSL stack management aren't needed as with certificates. Based on previous work by Mini Hawthorne. --- src/event/ngx_event_openssl.c | 154 +------------------------ src/event/ngx_event_openssl.h | 1 + src/event/ngx_event_openssl_cache.c | 168 ++++++++++++++++++++++++++++ 3 files changed, 172 insertions(+), 151 deletions(-) diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c index 018d0301627..1a6ca18adfc 100644 --- a/src/event/ngx_event_openssl.c +++ b/src/event/ngx_event_openssl.c @@ -18,10 +18,6 @@ typedef struct { } ngx_openssl_conf_t; -static EVP_PKEY *ngx_ssl_load_certificate_key(ngx_pool_t *pool, char **err, - ngx_str_t *key, ngx_array_t *passwords); -static int ngx_ssl_password_callback(char *buf, int size, int rwflag, - void *userdata); static int ngx_ssl_verify_callback(int ok, X509_STORE_CTX *x509_store); static void ngx_ssl_info_callback(const ngx_ssl_conn_t *ssl_conn, int where, int ret); @@ -537,7 +533,7 @@ ngx_ssl_certificate(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *cert, } #endif - pkey = ngx_ssl_load_certificate_key(cf->pool, &err, key, passwords); + pkey = ngx_ssl_cache_fetch(cf, NGX_SSL_CACHE_PKEY, &err, key, passwords); if (pkey == NULL) { if (err != NULL) { ngx_ssl_error(NGX_LOG_EMERG, ssl->log, 0, @@ -611,7 +607,8 @@ ngx_ssl_connection_certificate(ngx_connection_t *c, ngx_pool_t *pool, #endif - pkey = ngx_ssl_load_certificate_key(pool, &err, key, passwords); + pkey = ngx_ssl_cache_connection_fetch(pool, NGX_SSL_CACHE_PKEY, &err, + key, passwords); if (pkey == NULL) { if (err != NULL) { ngx_ssl_error(NGX_LOG_ERR, c->log, 0, @@ -635,151 +632,6 @@ ngx_ssl_connection_certificate(ngx_connection_t *c, ngx_pool_t *pool, } -static EVP_PKEY * -ngx_ssl_load_certificate_key(ngx_pool_t *pool, char **err, - ngx_str_t *key, ngx_array_t *passwords) -{ - BIO *bio; - EVP_PKEY *pkey; - ngx_str_t *pwd; - ngx_uint_t tries; - pem_password_cb *cb; - - if (ngx_strncmp(key->data, "engine:", sizeof("engine:") - 1) == 0) { - -#ifndef OPENSSL_NO_ENGINE - - u_char *p, *last; - ENGINE *engine; - - p = key->data + sizeof("engine:") - 1; - last = (u_char *) ngx_strchr(p, ':'); - - if (last == NULL) { - *err = "invalid syntax"; - return NULL; - } - - *last = '\0'; - - engine = ENGINE_by_id((char *) p); - - *last++ = ':'; - - if (engine == NULL) { - *err = "ENGINE_by_id() failed"; - return NULL; - } - - pkey = ENGINE_load_private_key(engine, (char *) last, 0, 0); - - if (pkey == NULL) { - *err = "ENGINE_load_private_key() failed"; - ENGINE_free(engine); - return NULL; - } - - ENGINE_free(engine); - - return pkey; - -#else - - *err = "loading \"engine:...\" certificate keys is not supported"; - return NULL; - -#endif - } - - if (ngx_strncmp(key->data, "data:", sizeof("data:") - 1) == 0) { - - bio = BIO_new_mem_buf(key->data + sizeof("data:") - 1, - key->len - (sizeof("data:") - 1)); - if (bio == NULL) { - *err = "BIO_new_mem_buf() failed"; - return NULL; - } - - } else { - - if (ngx_get_full_name(pool, (ngx_str_t *) &ngx_cycle->conf_prefix, key) - != NGX_OK) - { - *err = NULL; - return NULL; - } - - bio = BIO_new_file((char *) key->data, "r"); - if (bio == NULL) { - *err = "BIO_new_file() failed"; - return NULL; - } - } - - if (passwords) { - tries = passwords->nelts; - pwd = passwords->elts; - cb = ngx_ssl_password_callback; - - } else { - tries = 1; - pwd = NULL; - cb = NULL; - } - - for ( ;; ) { - - pkey = PEM_read_bio_PrivateKey(bio, NULL, cb, pwd); - if (pkey != NULL) { - break; - } - - if (tries-- > 1) { - ERR_clear_error(); - (void) BIO_reset(bio); - pwd++; - continue; - } - - *err = "PEM_read_bio_PrivateKey() failed"; - BIO_free(bio); - return NULL; - } - - BIO_free(bio); - - return pkey; -} - - -static int -ngx_ssl_password_callback(char *buf, int size, int rwflag, void *userdata) -{ - ngx_str_t *pwd = userdata; - - if (rwflag) { - ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, 0, - "ngx_ssl_password_callback() is called for encryption"); - return 0; - } - - if (pwd == NULL) { - return 0; - } - - if (pwd->len > (size_t) size) { - ngx_log_error(NGX_LOG_ERR, ngx_cycle->log, 0, - "password is truncated to %d bytes", size); - } else { - size = pwd->len; - } - - ngx_memcpy(buf, pwd->data, size); - - return size; -} - - ngx_int_t ngx_ssl_ciphers(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *ciphers, ngx_uint_t prefer_server_ciphers) diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h index addc7b4d3aa..5e36fb5e08c 100644 --- a/src/event/ngx_event_openssl.h +++ b/src/event/ngx_event_openssl.h @@ -194,6 +194,7 @@ typedef struct { #define NGX_SSL_CACHE_CERT 0 +#define NGX_SSL_CACHE_PKEY 1 ngx_int_t ngx_ssl_init(ngx_log_t *log); diff --git a/src/event/ngx_event_openssl_cache.c b/src/event/ngx_event_openssl_cache.c index 20b87b79a86..f5bb9a2419d 100644 --- a/src/event/ngx_event_openssl_cache.c +++ b/src/event/ngx_event_openssl_cache.c @@ -11,6 +11,7 @@ #define NGX_SSL_CACHE_PATH 0 #define NGX_SSL_CACHE_DATA 1 +#define NGX_SSL_CACHE_ENGINE 2 typedef struct { @@ -57,6 +58,13 @@ static void *ngx_ssl_cache_cert_create(ngx_ssl_cache_key_t *id, char **err, static void ngx_ssl_cache_cert_free(void *data); static void *ngx_ssl_cache_cert_ref(char **err, void *data); +static void *ngx_ssl_cache_pkey_create(ngx_ssl_cache_key_t *id, char **err, + void *data); +static int ngx_ssl_cache_pkey_password_callback(char *buf, int size, int rwflag, + void *userdata); +static void ngx_ssl_cache_pkey_free(void *data); +static void *ngx_ssl_cache_pkey_ref(char **err, void *data); + static BIO *ngx_ssl_cache_create_bio(ngx_ssl_cache_key_t *id, char **err); static void *ngx_openssl_cache_create_conf(ngx_cycle_t *cycle); @@ -94,6 +102,11 @@ static ngx_ssl_cache_type_t ngx_ssl_cache_types[] = { { ngx_ssl_cache_cert_create, ngx_ssl_cache_cert_free, ngx_ssl_cache_cert_ref }, + + /* NGX_SSL_CACHE_PKEY */ + { ngx_ssl_cache_pkey_create, + ngx_ssl_cache_pkey_free, + ngx_ssl_cache_pkey_ref }, }; @@ -167,6 +180,11 @@ ngx_ssl_cache_init_key(ngx_pool_t *pool, ngx_uint_t index, ngx_str_t *path, if (ngx_strncmp(path->data, "data:", sizeof("data:") - 1) == 0) { id->type = NGX_SSL_CACHE_DATA; + } else if (index == NGX_SSL_CACHE_PKEY + && ngx_strncmp(path->data, "engine:", sizeof("engine:") - 1) == 0) + { + id->type = NGX_SSL_CACHE_ENGINE; + } else { if (ngx_get_full_name(pool, (ngx_str_t *) &ngx_cycle->conf_prefix, path) != NGX_OK) @@ -349,6 +367,156 @@ ngx_ssl_cache_cert_ref(char **err, void *data) } +static void * +ngx_ssl_cache_pkey_create(ngx_ssl_cache_key_t *id, char **err, void *data) +{ + ngx_array_t *passwords = data; + + BIO *bio; + EVP_PKEY *pkey; + ngx_str_t *pwd; + ngx_uint_t tries; + pem_password_cb *cb; + + if (id->type == NGX_SSL_CACHE_ENGINE) { + +#ifndef OPENSSL_NO_ENGINE + + u_char *p, *last; + ENGINE *engine; + + p = id->data + sizeof("engine:") - 1; + last = (u_char *) ngx_strchr(p, ':'); + + if (last == NULL) { + *err = "invalid syntax"; + return NULL; + } + + *last = '\0'; + + engine = ENGINE_by_id((char *) p); + + *last++ = ':'; + + if (engine == NULL) { + *err = "ENGINE_by_id() failed"; + return NULL; + } + + pkey = ENGINE_load_private_key(engine, (char *) last, 0, 0); + + if (pkey == NULL) { + *err = "ENGINE_load_private_key() failed"; + ENGINE_free(engine); + return NULL; + } + + ENGINE_free(engine); + + return pkey; + +#else + + *err = "loading \"engine:...\" certificate keys is not supported"; + return NULL; + +#endif + } + + bio = ngx_ssl_cache_create_bio(id, err); + if (bio == NULL) { + return NULL; + } + + if (passwords) { + tries = passwords->nelts; + pwd = passwords->elts; + cb = ngx_ssl_cache_pkey_password_callback; + + } else { + tries = 1; + pwd = NULL; + cb = NULL; + } + + for ( ;; ) { + + pkey = PEM_read_bio_PrivateKey(bio, NULL, cb, pwd); + if (pkey != NULL) { + break; + } + + if (tries-- > 1) { + ERR_clear_error(); + (void) BIO_reset(bio); + pwd++; + continue; + } + + *err = "PEM_read_bio_PrivateKey() failed"; + BIO_free(bio); + return NULL; + } + + BIO_free(bio); + + return pkey; +} + + +static int +ngx_ssl_cache_pkey_password_callback(char *buf, int size, int rwflag, + void *userdata) +{ + ngx_str_t *pwd = userdata; + + if (rwflag) { + ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, 0, + "ngx_ssl_cache_pkey_password_callback() is called " + "for encryption"); + return 0; + } + + if (pwd == NULL) { + return 0; + } + + if (pwd->len > (size_t) size) { + ngx_log_error(NGX_LOG_ERR, ngx_cycle->log, 0, + "password is truncated to %d bytes", size); + } else { + size = pwd->len; + } + + ngx_memcpy(buf, pwd->data, size); + + return size; +} + + +static void +ngx_ssl_cache_pkey_free(void *data) +{ + EVP_PKEY_free(data); +} + + +static void * +ngx_ssl_cache_pkey_ref(char **err, void *data) +{ + EVP_PKEY *pkey = data; + +#if (OPENSSL_VERSION_NUMBER >= 0x10100000L) + EVP_PKEY_up_ref(pkey); +#else + CRYPTO_add(&pkey->references, 1, CRYPTO_LOCK_EVP_PKEY); +#endif + + return data; +} + + static BIO * ngx_ssl_cache_create_bio(ngx_ssl_cache_key_t *id, char **err) { From 61314518de74fcb3af954ea6e6cb2820307676d0 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Mon, 9 Sep 2024 19:05:31 +0400 Subject: [PATCH 172/279] SSL: caching CRLs. Based on previous work by Mini Hawthorne. --- src/event/ngx_event_openssl.c | 72 ++++++++++++++----- src/event/ngx_event_openssl.h | 1 + src/event/ngx_event_openssl_cache.c | 108 +++++++++++++++++++++++++++- 3 files changed, 164 insertions(+), 17 deletions(-) diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c index 1a6ca18adfc..72189e1a4cb 100644 --- a/src/event/ngx_event_openssl.c +++ b/src/event/ngx_event_openssl.c @@ -18,6 +18,7 @@ typedef struct { } ngx_openssl_conf_t; +static ngx_inline ngx_int_t ngx_ssl_cert_already_in_hash(void); static int ngx_ssl_verify_callback(int ok, X509_STORE_CTX *x509_store); static void ngx_ssl_info_callback(const ngx_ssl_conn_t *ssl_conn, int where, int ret); @@ -739,17 +740,16 @@ ngx_ssl_trusted_certificate(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *cert, ngx_int_t ngx_ssl_crl(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *crl) { - X509_STORE *store; - X509_LOOKUP *lookup; + int n, i; + char *err; + X509_CRL *x509; + X509_STORE *store; + STACK_OF(X509_CRL) *chain; if (crl->len == 0) { return NGX_OK; } - if (ngx_conf_full_name(cf->cycle, crl, 1) != NGX_OK) { - return NGX_ERROR; - } - store = SSL_CTX_get_cert_store(ssl->ctx); if (store == NULL) { @@ -758,22 +758,36 @@ ngx_ssl_crl(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *crl) return NGX_ERROR; } - lookup = X509_STORE_add_lookup(store, X509_LOOKUP_file()); + chain = ngx_ssl_cache_fetch(cf, NGX_SSL_CACHE_CRL, &err, crl, NULL); + if (chain == NULL) { + if (err != NULL) { + ngx_ssl_error(NGX_LOG_EMERG, ssl->log, 0, + "cannot load CRL \"%s\": %s", crl->data, err); + } - if (lookup == NULL) { - ngx_ssl_error(NGX_LOG_EMERG, ssl->log, 0, - "X509_STORE_add_lookup() failed"); return NGX_ERROR; } - if (X509_LOOKUP_load_file(lookup, (char *) crl->data, X509_FILETYPE_PEM) - == 0) - { - ngx_ssl_error(NGX_LOG_EMERG, ssl->log, 0, - "X509_LOOKUP_load_file(\"%s\") failed", crl->data); - return NGX_ERROR; + n = sk_X509_CRL_num(chain); + + for (i = 0; i < n; i++) { + x509 = sk_X509_CRL_value(chain, i); + + if (X509_STORE_add_crl(store, x509) != 1) { + + if (ngx_ssl_cert_already_in_hash()) { + continue; + } + + ngx_ssl_error(NGX_LOG_EMERG, ssl->log, 0, + "X509_STORE_add_crl(\"%s\") failed", crl->data); + sk_X509_CRL_pop_free(chain, X509_CRL_free); + return NGX_ERROR; + } } + sk_X509_CRL_pop_free(chain, X509_CRL_free); + X509_STORE_set_flags(store, X509_V_FLAG_CRL_CHECK|X509_V_FLAG_CRL_CHECK_ALL); @@ -781,6 +795,32 @@ ngx_ssl_crl(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *crl) } +static ngx_inline ngx_int_t +ngx_ssl_cert_already_in_hash(void) +{ +#if !(OPENSSL_VERSION_NUMBER >= 0x1010009fL \ + || LIBRESSL_VERSION_NUMBER >= 0x3050000fL) + u_long error; + + /* + * OpenSSL prior to 1.1.0i doesn't ignore duplicate certificate entries, + * see https://github.com/openssl/openssl/commit/c0452248 + */ + + error = ERR_peek_last_error(); + + if (ERR_GET_LIB(error) == ERR_LIB_X509 + && ERR_GET_REASON(error) == X509_R_CERT_ALREADY_IN_HASH_TABLE) + { + ERR_clear_error(); + return 1; + } +#endif + + return 0; +} + + static int ngx_ssl_verify_callback(int ok, X509_STORE_CTX *x509_store) { diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h index 5e36fb5e08c..e2f0e5821ae 100644 --- a/src/event/ngx_event_openssl.h +++ b/src/event/ngx_event_openssl.h @@ -195,6 +195,7 @@ typedef struct { #define NGX_SSL_CACHE_CERT 0 #define NGX_SSL_CACHE_PKEY 1 +#define NGX_SSL_CACHE_CRL 2 ngx_int_t ngx_ssl_init(ngx_log_t *log); diff --git a/src/event/ngx_event_openssl_cache.c b/src/event/ngx_event_openssl_cache.c index f5bb9a2419d..b2615d2bfce 100644 --- a/src/event/ngx_event_openssl_cache.c +++ b/src/event/ngx_event_openssl_cache.c @@ -65,6 +65,11 @@ static int ngx_ssl_cache_pkey_password_callback(char *buf, int size, int rwflag, static void ngx_ssl_cache_pkey_free(void *data); static void *ngx_ssl_cache_pkey_ref(char **err, void *data); +static void *ngx_ssl_cache_crl_create(ngx_ssl_cache_key_t *id, char **err, + void *data); +static void ngx_ssl_cache_crl_free(void *data); +static void *ngx_ssl_cache_crl_ref(char **err, void *data); + static BIO *ngx_ssl_cache_create_bio(ngx_ssl_cache_key_t *id, char **err); static void *ngx_openssl_cache_create_conf(ngx_cycle_t *cycle); @@ -107,6 +112,11 @@ static ngx_ssl_cache_type_t ngx_ssl_cache_types[] = { { ngx_ssl_cache_pkey_create, ngx_ssl_cache_pkey_free, ngx_ssl_cache_pkey_ref }, + + /* NGX_SSL_CACHE_CRL */ + { ngx_ssl_cache_crl_create, + ngx_ssl_cache_crl_free, + ngx_ssl_cache_crl_ref }, }; @@ -177,7 +187,9 @@ static ngx_int_t ngx_ssl_cache_init_key(ngx_pool_t *pool, ngx_uint_t index, ngx_str_t *path, ngx_ssl_cache_key_t *id) { - if (ngx_strncmp(path->data, "data:", sizeof("data:") - 1) == 0) { + if (index <= NGX_SSL_CACHE_PKEY + && ngx_strncmp(path->data, "data:", sizeof("data:") - 1) == 0) + { id->type = NGX_SSL_CACHE_DATA; } else if (index == NGX_SSL_CACHE_PKEY @@ -517,6 +529,100 @@ ngx_ssl_cache_pkey_ref(char **err, void *data) } +static void * +ngx_ssl_cache_crl_create(ngx_ssl_cache_key_t *id, char **err, void *data) +{ + BIO *bio; + u_long n; + X509_CRL *x509; + STACK_OF(X509_CRL) *chain; + + chain = sk_X509_CRL_new_null(); + if (chain == NULL) { + *err = "sk_X509_CRL_new_null() failed"; + return NULL; + } + + bio = ngx_ssl_cache_create_bio(id, err); + if (bio == NULL) { + sk_X509_CRL_pop_free(chain, X509_CRL_free); + return NULL; + } + + for ( ;; ) { + + x509 = PEM_read_bio_X509_CRL(bio, NULL, NULL, NULL); + if (x509 == NULL) { + n = ERR_peek_last_error(); + + if (ERR_GET_LIB(n) == ERR_LIB_PEM + && ERR_GET_REASON(n) == PEM_R_NO_START_LINE + && sk_X509_CRL_num(chain) > 0) + { + /* end of file */ + ERR_clear_error(); + break; + } + + /* some real error */ + + *err = "PEM_read_bio_X509_CRL() failed"; + BIO_free(bio); + sk_X509_CRL_pop_free(chain, X509_CRL_free); + return NULL; + } + + if (sk_X509_CRL_push(chain, x509) == 0) { + *err = "sk_X509_CRL_push() failed"; + BIO_free(bio); + X509_CRL_free(x509); + sk_X509_CRL_pop_free(chain, X509_CRL_free); + return NULL; + } + } + + BIO_free(bio); + + return chain; +} + + +static void +ngx_ssl_cache_crl_free(void *data) +{ + sk_X509_CRL_pop_free(data, X509_CRL_free); +} + + +static void * +ngx_ssl_cache_crl_ref(char **err, void *data) +{ + int n, i; + X509_CRL *x509; + STACK_OF(X509_CRL) *chain; + + chain = sk_X509_CRL_dup(data); + if (chain == NULL) { + *err = "sk_X509_CRL_dup() failed"; + return NULL; + } + + n = sk_X509_CRL_num(chain); + + for (i = 0; i < n; i++) { + x509 = sk_X509_CRL_value(chain, i); + +#if (OPENSSL_VERSION_NUMBER >= 0x10100000L) + X509_CRL_up_ref(x509); +#else + CRYPTO_add(&x509->references, 1, CRYPTO_LOCK_X509_CRL); +#endif + } + + return chain; +} + + static BIO * ngx_ssl_cache_create_bio(ngx_ssl_cache_key_t *id, char **err) { From 5917e9de5a45bb7288c1c433db840d1a4c6290f3 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Mon, 9 Sep 2024 19:05:58 +0400 Subject: [PATCH 173/279] SSL: caching CA certificates. This can potentially provide a large amount of savings, because CA certificates can be quite large. Based on previous work by Mini Hawthorne. --- src/event/ngx_event_openssl.c | 146 ++++++++++++++++++++++------ src/event/ngx_event_openssl.h | 1 + src/event/ngx_event_openssl_cache.c | 66 +++++++++++++ 3 files changed, 185 insertions(+), 28 deletions(-) diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c index 72189e1a4cb..2b1d107df53 100644 --- a/src/event/ngx_event_openssl.c +++ b/src/event/ngx_event_openssl.c @@ -22,6 +22,8 @@ static ngx_inline ngx_int_t ngx_ssl_cert_already_in_hash(void); static int ngx_ssl_verify_callback(int ok, X509_STORE_CTX *x509_store); static void ngx_ssl_info_callback(const ngx_ssl_conn_t *ssl_conn, int where, int ret); +static int ngx_ssl_cmp_x509_name(const X509_NAME *const *a, + const X509_NAME *const *b); static void ngx_ssl_passwords_cleanup(void *data); static int ngx_ssl_new_client_session(ngx_ssl_conn_t *ssl_conn, ngx_ssl_session_t *sess); @@ -656,6 +658,12 @@ ngx_int_t ngx_ssl_client_certificate(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *cert, ngx_int_t depth) { + int n, i; + char *err; + X509 *x509; + X509_NAME *name; + X509_STORE *store; + STACK_OF(X509) *chain; STACK_OF(X509_NAME) *list; SSL_CTX_set_verify(ssl->ctx, SSL_VERIFY_PEER, ngx_ssl_verify_callback); @@ -666,34 +674,84 @@ ngx_ssl_client_certificate(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *cert, return NGX_OK; } - if (ngx_conf_full_name(cf->cycle, cert, 1) != NGX_OK) { + list = sk_X509_NAME_new(ngx_ssl_cmp_x509_name); + if (list == NULL) { return NGX_ERROR; } - if (SSL_CTX_load_verify_locations(ssl->ctx, (char *) cert->data, NULL) - == 0) - { + store = SSL_CTX_get_cert_store(ssl->ctx); + + if (store == NULL) { ngx_ssl_error(NGX_LOG_EMERG, ssl->log, 0, - "SSL_CTX_load_verify_locations(\"%s\") failed", - cert->data); + "SSL_CTX_get_cert_store() failed"); return NGX_ERROR; } - /* - * SSL_CTX_load_verify_locations() may leave errors in the error queue - * while returning success - */ + chain = ngx_ssl_cache_fetch(cf, NGX_SSL_CACHE_CA, &err, cert, NULL); + if (chain == NULL) { + if (err != NULL) { + ngx_ssl_error(NGX_LOG_EMERG, ssl->log, 0, + "cannot load certificate \"%s\": %s", + cert->data, err); + } - ERR_clear_error(); + sk_X509_NAME_pop_free(list, X509_NAME_free); + return NGX_ERROR; + } - list = SSL_load_client_CA_file((char *) cert->data); + n = sk_X509_num(chain); - if (list == NULL) { - ngx_ssl_error(NGX_LOG_EMERG, ssl->log, 0, - "SSL_load_client_CA_file(\"%s\") failed", cert->data); - return NGX_ERROR; + for (i = 0; i < n; i++) { + x509 = sk_X509_value(chain, i); + + if (X509_STORE_add_cert(store, x509) != 1) { + + if (ngx_ssl_cert_already_in_hash()) { + continue; + } + + ngx_ssl_error(NGX_LOG_EMERG, ssl->log, 0, + "X509_STORE_add_cert(\"%s\") failed", cert->data); + sk_X509_NAME_pop_free(list, X509_NAME_free); + sk_X509_pop_free(chain, X509_free); + return NGX_ERROR; + } + + name = X509_get_subject_name(x509); + if (name == NULL) { + ngx_ssl_error(NGX_LOG_EMERG, ssl->log, 0, + "X509_get_subject_name(\"%s\") failed", cert->data); + sk_X509_NAME_pop_free(list, X509_NAME_free); + sk_X509_pop_free(chain, X509_free); + return NGX_ERROR; + } + + name = X509_NAME_dup(name); + if (name == NULL) { + sk_X509_NAME_pop_free(list, X509_NAME_free); + sk_X509_pop_free(chain, X509_free); + return NGX_ERROR; + } + +#ifdef OPENSSL_IS_BORINGSSL + if (sk_X509_NAME_find(list, NULL, name) > 0) { +#else + if (sk_X509_NAME_find(list, name) >= 0) { +#endif + X509_NAME_free(name); + continue; + } + + if (sk_X509_NAME_push(list, name) == 0) { + sk_X509_NAME_pop_free(list, X509_NAME_free); + sk_X509_pop_free(chain, X509_free); + X509_NAME_free(name); + return NGX_ERROR; + } } + sk_X509_pop_free(chain, X509_free); + SSL_CTX_set_client_CA_list(ssl->ctx, list); return NGX_OK; @@ -704,6 +762,12 @@ ngx_int_t ngx_ssl_trusted_certificate(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *cert, ngx_int_t depth) { + int i, n; + char *err; + X509 *x509; + X509_STORE *store; + STACK_OF(X509) *chain; + SSL_CTX_set_verify(ssl->ctx, SSL_CTX_get_verify_mode(ssl->ctx), ngx_ssl_verify_callback); @@ -713,25 +777,44 @@ ngx_ssl_trusted_certificate(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *cert, return NGX_OK; } - if (ngx_conf_full_name(cf->cycle, cert, 1) != NGX_OK) { + store = SSL_CTX_get_cert_store(ssl->ctx); + + if (store == NULL) { + ngx_ssl_error(NGX_LOG_EMERG, ssl->log, 0, + "SSL_CTX_get_cert_store() failed"); return NGX_ERROR; } - if (SSL_CTX_load_verify_locations(ssl->ctx, (char *) cert->data, NULL) - == 0) - { - ngx_ssl_error(NGX_LOG_EMERG, ssl->log, 0, - "SSL_CTX_load_verify_locations(\"%s\") failed", - cert->data); + chain = ngx_ssl_cache_fetch(cf, NGX_SSL_CACHE_CA, &err, cert, NULL); + if (chain == NULL) { + if (err != NULL) { + ngx_ssl_error(NGX_LOG_EMERG, ssl->log, 0, + "cannot load certificate \"%s\": %s", + cert->data, err); + } + return NGX_ERROR; } - /* - * SSL_CTX_load_verify_locations() may leave errors in the error queue - * while returning success - */ + n = sk_X509_num(chain); - ERR_clear_error(); + for (i = 0; i < n; i++) { + x509 = sk_X509_value(chain, i); + + if (X509_STORE_add_cert(store, x509) != 1) { + + if (ngx_ssl_cert_already_in_hash()) { + continue; + } + + ngx_ssl_error(NGX_LOG_EMERG, ssl->log, 0, + "X509_STORE_add_cert(\"%s\") failed", cert->data); + sk_X509_pop_free(chain, X509_free); + return NGX_ERROR; + } + } + + sk_X509_pop_free(chain, X509_free); return NGX_OK; } @@ -987,6 +1070,13 @@ ngx_ssl_info_callback(const ngx_ssl_conn_t *ssl_conn, int where, int ret) } +static int +ngx_ssl_cmp_x509_name(const X509_NAME *const *a, const X509_NAME *const *b) +{ + return (X509_NAME_cmp(*a, *b)); +} + + ngx_array_t * ngx_ssl_read_password_file(ngx_conf_t *cf, ngx_str_t *file) { diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h index e2f0e5821ae..6d171229c6f 100644 --- a/src/event/ngx_event_openssl.h +++ b/src/event/ngx_event_openssl.h @@ -196,6 +196,7 @@ typedef struct { #define NGX_SSL_CACHE_CERT 0 #define NGX_SSL_CACHE_PKEY 1 #define NGX_SSL_CACHE_CRL 2 +#define NGX_SSL_CACHE_CA 3 ngx_int_t ngx_ssl_init(ngx_log_t *log); diff --git a/src/event/ngx_event_openssl_cache.c b/src/event/ngx_event_openssl_cache.c index b2615d2bfce..f43bdb5e738 100644 --- a/src/event/ngx_event_openssl_cache.c +++ b/src/event/ngx_event_openssl_cache.c @@ -70,6 +70,9 @@ static void *ngx_ssl_cache_crl_create(ngx_ssl_cache_key_t *id, char **err, static void ngx_ssl_cache_crl_free(void *data); static void *ngx_ssl_cache_crl_ref(char **err, void *data); +static void *ngx_ssl_cache_ca_create(ngx_ssl_cache_key_t *id, char **err, + void *data); + static BIO *ngx_ssl_cache_create_bio(ngx_ssl_cache_key_t *id, char **err); static void *ngx_openssl_cache_create_conf(ngx_cycle_t *cycle); @@ -117,6 +120,11 @@ static ngx_ssl_cache_type_t ngx_ssl_cache_types[] = { { ngx_ssl_cache_crl_create, ngx_ssl_cache_crl_free, ngx_ssl_cache_crl_ref }, + + /* NGX_SSL_CACHE_CA */ + { ngx_ssl_cache_ca_create, + ngx_ssl_cache_cert_free, + ngx_ssl_cache_cert_ref } }; @@ -623,6 +631,64 @@ ngx_ssl_cache_crl_ref(char **err, void *data) } +static void * +ngx_ssl_cache_ca_create(ngx_ssl_cache_key_t *id, char **err, void *data) +{ + BIO *bio; + X509 *x509; + u_long n; + STACK_OF(X509) *chain; + + chain = sk_X509_new_null(); + if (chain == NULL) { + *err = "sk_X509_new_null() failed"; + return NULL; + } + + bio = ngx_ssl_cache_create_bio(id, err); + if (bio == NULL) { + sk_X509_pop_free(chain, X509_free); + return NULL; + } + + for ( ;; ) { + + x509 = PEM_read_bio_X509_AUX(bio, NULL, NULL, NULL); + if (x509 == NULL) { + n = ERR_peek_last_error(); + + if (ERR_GET_LIB(n) == ERR_LIB_PEM + && ERR_GET_REASON(n) == PEM_R_NO_START_LINE + && sk_X509_num(chain) > 0) + { + /* end of file */ + ERR_clear_error(); + break; + } + + /* some real error */ + + *err = "PEM_read_bio_X509_AUX() failed"; + BIO_free(bio); + sk_X509_pop_free(chain, X509_free); + return NULL; + } + + if (sk_X509_push(chain, x509) == 0) { + *err = "sk_X509_push() failed"; + BIO_free(bio); + X509_free(x509); + sk_X509_pop_free(chain, X509_free); + return NULL; + } + } + + BIO_free(bio); + + return chain; +} + + static BIO * ngx_ssl_cache_create_bio(ngx_ssl_cache_key_t *id, char **err) { From 0e7c9ddb275a496b24adc3f1de8eebe70a40ee26 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Tue, 1 Oct 2024 19:23:45 +0400 Subject: [PATCH 174/279] Updated OpenSSL used for win32 builds. --- misc/GNUmakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/GNUmakefile b/misc/GNUmakefile index fcf34ec34a5..e18a86a63aa 100644 --- a/misc/GNUmakefile +++ b/misc/GNUmakefile @@ -6,7 +6,7 @@ TEMP = tmp CC = cl OBJS = objs.msvc8 -OPENSSL = openssl-3.0.14 +OPENSSL = openssl-3.0.15 ZLIB = zlib-1.3.1 PCRE = pcre2-10.39 From e24f7ccc161f1a2a759eb27263ec9af4fc7c8e96 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Tue, 1 Oct 2024 23:59:48 +0400 Subject: [PATCH 175/279] nginx-1.27.2-RELEASE --- docs/xml/nginx/changes.xml | 65 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/docs/xml/nginx/changes.xml b/docs/xml/nginx/changes.xml index 9c2f317bc25..c18dedff43c 100644 --- a/docs/xml/nginx/changes.xml +++ b/docs/xml/nginx/changes.xml @@ -5,6 +5,71 @@ + + + + +SSL-сертификаты, секретные ключи и списки CRL теперь кешируются +на старте или во время переконфигурации. + + +SSL certificates, secret keys, and CRLs are now cached +on start or during reconfiguration. + + + + + +проверка клиентских сертификатов с помощью OCSP в модуле stream. + + +client certificate validation with OCSP in the stream module. + + + + + +поддержка OCSP stapling в модуле stream. + + +OCSP stapling support in the stream module. + + + + + +директива proxy_pass_trailers в модуле ngx_http_proxy_module. + + +the "proxy_pass_trailers" directive in the ngx_http_proxy_module. + + + + + +директива ssl_client_certificate теперь поддерживает сертификаты +с дополнительными данными. + + +the "ssl_client_certificate" directive now supports certificates +with auxiliary information. + + + + + +теперь наличие директивы ssl_client_certificate не обязательно +для проверки клиентских SSL-сертификатов. + + +now the "ssl_client_certificate" directive is not required +for client SSL certificates verification. + + + + + + From 144778aee6e377c9bcd84368848b2e27871d570f Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Mon, 7 Oct 2024 19:10:39 +0400 Subject: [PATCH 176/279] Version bump. --- src/core/nginx.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/nginx.h b/src/core/nginx.h index 140a2940874..fb6fa47370a 100644 --- a/src/core/nginx.h +++ b/src/core/nginx.h @@ -9,8 +9,8 @@ #define _NGINX_H_INCLUDED_ -#define nginx_version 1027002 -#define NGINX_VERSION "1.27.2" +#define nginx_version 1027003 +#define NGINX_VERSION "1.27.3" #define NGINX_VER "nginx/" NGINX_VERSION #ifdef NGX_BUILD From 3f6d94d8881ca6755b1baa4dd6248a7b7ed15735 Mon Sep 17 00:00:00 2001 From: nandsky Date: Mon, 30 Sep 2024 20:51:17 +0800 Subject: [PATCH 177/279] QUIC: prevent deleted stream frame retransmissions. Since a2a513b93cae, stream frames no longer need to be retransmitted after it was deleted. The frames which were retransmitted before, could be stream data frames sent prior to a RESET_STREAM. Such retransmissions are explicitly prohibited by RFC 9000, Section 19.4. --- src/event/quic/ngx_event_quic_ack.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/event/quic/ngx_event_quic_ack.c b/src/event/quic/ngx_event_quic_ack.c index c7ffd44dd77..c953b8042ec 100644 --- a/src/event/quic/ngx_event_quic_ack.c +++ b/src/event/quic/ngx_event_quic_ack.c @@ -631,13 +631,12 @@ ngx_quic_resend_frames(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx) case NGX_QUIC_FT_STREAM: qs = ngx_quic_find_stream(&qc->streams.tree, f->u.stream.stream_id); - if (qs) { - if (qs->send_state == NGX_QUIC_STREAM_SEND_RESET_SENT - || qs->send_state == NGX_QUIC_STREAM_SEND_RESET_RECVD) - { - ngx_quic_free_frame(c, f); - break; - } + if (qs == NULL + || qs->send_state == NGX_QUIC_STREAM_SEND_RESET_SENT + || qs->send_state == NGX_QUIC_STREAM_SEND_RESET_RECVD) + { + ngx_quic_free_frame(c, f); + break; } /* fall through */ From b394d44cfa7e5c2d48a174d06f4b899b6cfd3ccf Mon Sep 17 00:00:00 2001 From: Thierry Bastian Date: Wed, 9 Oct 2024 09:18:49 +0200 Subject: [PATCH 178/279] Configure: MSVC compatibility with PCRE2 10.43. --- auto/lib/pcre/make | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/auto/lib/pcre/make b/auto/lib/pcre/make index 839ef294bb3..182590ac521 100644 --- a/auto/lib/pcre/make +++ b/auto/lib/pcre/make @@ -36,7 +36,8 @@ if [ $PCRE_LIBRARY = PCRE2 ]; then pcre2_valid_utf.c \ pcre2_xclass.c" - ngx_pcre_test="pcre2_convert.c \ + ngx_pcre_test="pcre2_chkdint.c \ + pcre2_convert.c \ pcre2_extuni.c \ pcre2_find_bracket.c \ pcre2_script_run.c \ From f45c2707ea1bf6fde3bd0c045cc4a63cdc4a966a Mon Sep 17 00:00:00 2001 From: jzebor-at-f5 <144731595+jzebor-at-f5@users.noreply.github.com> Date: Fri, 11 Oct 2024 09:48:53 -0700 Subject: [PATCH 179/279] Updated security policy to include disclosure details. --- SECURITY.md | 94 ++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 89 insertions(+), 5 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index 2b48e47e3ae..f5cfcd7885f 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,8 +1,8 @@ # Security Policy -## Latest Versions - -We advise users to run the most recent mainline or stable release of nginx. +This document provides an overview of security concerns related to nginx +deployments, focusing on confidentiality, integrity, availability, and the +implications of configurations and misconfigurations. ## Reporting a Vulnerability @@ -10,11 +10,95 @@ Please report any vulnerabilities via one of the following methods (in order of preference): 1. [Report a vulnerability](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing-information-about-vulnerabilities/privately-reporting-a-security-vulnerability) -within this repository. We are using the Github workflow that allows us to -manage vulnerabilities in a private manner and to interact with reporters +within this repository. We are using the GitHub workflow that allows us to +manage vulnerabilities in a private manner and interact with reporters securely. 2. [Report directly to F5](https://www.f5.com/services/support/report-a-vulnerability). 3. Report via email to security-alert@nginx.org. This method will be deprecated in the future. + +### Vulnerability Disclosure and Fix Process + +The nginx team expects that all suspected vulnerabilities be reported +privately via the +[Reporting a Vulnerability](SECURITY.md#reporting-a-vulnerability) guidelines. +If a publicly released vulnerability is reported, we +may request to handle it according to the private disclosure process. +If the reporter agrees, we will follow the private disclosure process. + +Security fixes will be applied to all supported stable releases, as well +as the mainline version, as applicable. We recommend using the most recent +mainline or stable release of nginx. Fixes are created and tested by the core +team using a GitHub private fork for security. If necessary, the reporter +may be invited to contribute to the fork and assist with the solution. + +The nginx team is committed to responsible information disclosure with +sufficient detail, such as the CVSS score and vector. Privately disclosed +vulnerabilities are embargoed by default until the fix is released. +Communications and fixes remain private until made public. As nginx is +supported by F5, we generally follow the +[F5 security vulnerability response policy](https://my.f5.com/manage/s/article/K4602). + +### Vulnerability Disclosure and Fix Service Level Objectives + +- We will acknowledge all vulnerability reports within 1 to 3 days. +- Fixes will be developed and released within 90 days from the date of +disclosure. If an extension is needed, we will work with the disclosing person. +- Publicly disclosed (i.e., Zero-Day vulnerabilities) will be addressed ASAP. + +## Confidentiality, Integrity, and Availability + +### Confidentiality and Integrity + +Vulnerabilities compromising data confidentiality or integrity are considered +the highest priority. Any issue leading to unauthorized data access, leaks, or +manipulation will trigger the security release process. + +### Availability + +Availability issues must meet the following criteria to trigger the security +release process: +- Is present in a standard module included with nginx. +- Arises from traffic that the module is designed to handle. +- Resource exhaustion issues are not mitigated by existing timeout, rate +limiting, or buffer size configurations, or applying changes is impractical. +- Results in highly asymmetric, extreme resource consumption. + +Availability issues excluded from the security release process: +- Local file content or upstream response content resulting only in worker +process termination. +- Issues with experimental features which result only in worker process +termination. + +## Trusted Configurations and Misconfigurations + +In nginx, configuration files, modules, certificate/key pairs, nginx JavaScript, +and local file content are considered trusted sources. Issues arising from +loading or execution of these trusted components are not considered +vulnerabilities. Operators are responsible for securing and maintaining the +integrity of these sources. Misconfigurations can create vulnerabilities, and +operators should implement configurations according to best practices, review +them regularly, and apply security updates. + +## Data Plane vs. Control Plane + +The data plane handles traffic through nginx, directly interacting with user +data. nginx inherently trusts the content and instructions from upstream +servers. The control plane governs configuration, management, and orchestration. +Misconfigurations or vulnerabilities in the control plane can cause improper +behavior in the data plane. + +## Modules Under Scope + +The policy applies to all nginx modules included in this repository. Security +considerations and attack vectors for each module will be identified, with +recommended configurations to mitigate risks. + +## Debug Logging and Core Files + +Debug logs and core files produced by nginx may contain un-sanitized data, +including sensitive information like client requests, server configurations, +and private key material. These artifacts must be handled carefully to avoid +exposing confidential data. From ebd18ec1812bd6f3de54d9f9fc81563a0ec9f264 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Wed, 9 Oct 2024 20:28:00 +0400 Subject: [PATCH 180/279] SSL: disabled TLSv1 and TLSv1.1 by default. TLSv1 and TLSv1.1 are formally deprecated and forbidden to negotiate due to insufficient security reasons outlined in RFC 8996. TLSv1 and TLSv1.1 are disabled in BoringSSL e95b0cad9 and LibreSSL 3.8.1 in the way they cannot be enabled in nginx configuration. In OpenSSL 3.0, they are only permitted at security level 0 (disabled by default). The support is dropped in Chrome 84, Firefox 78, and deprecated in Safari. This change disables TLSv1 and TLSv1.1 by default for OpenSSL 1.0.1 and newer, where TLSv1.2 support is available. For older library versions, which do not have alternatives, these protocol versions remain enabled. --- src/http/modules/ngx_http_grpc_module.c | 2 ++ src/http/modules/ngx_http_proxy_module.c | 2 ++ src/http/modules/ngx_http_ssl_module.c | 2 ++ src/http/modules/ngx_http_uwsgi_module.c | 2 ++ src/mail/ngx_mail_ssl_module.c | 2 ++ src/stream/ngx_stream_proxy_module.c | 2 ++ src/stream/ngx_stream_ssl_module.c | 2 ++ 7 files changed, 14 insertions(+) diff --git a/src/http/modules/ngx_http_grpc_module.c b/src/http/modules/ngx_http_grpc_module.c index e7726f3142d..d9456843dca 100644 --- a/src/http/modules/ngx_http_grpc_module.c +++ b/src/http/modules/ngx_http_grpc_module.c @@ -4477,7 +4477,9 @@ ngx_http_grpc_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) ngx_conf_merge_bitmask_value(conf->ssl_protocols, prev->ssl_protocols, (NGX_CONF_BITMASK_SET +#ifndef SSL_OP_NO_TLSv1_2 |NGX_SSL_TLSv1|NGX_SSL_TLSv1_1 +#endif |NGX_SSL_TLSv1_2|NGX_SSL_TLSv1_3)); ngx_conf_merge_str_value(conf->ssl_ciphers, prev->ssl_ciphers, diff --git a/src/http/modules/ngx_http_proxy_module.c b/src/http/modules/ngx_http_proxy_module.c index f9a373744ad..fe1952748ad 100644 --- a/src/http/modules/ngx_http_proxy_module.c +++ b/src/http/modules/ngx_http_proxy_module.c @@ -3944,7 +3944,9 @@ ngx_http_proxy_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) ngx_conf_merge_bitmask_value(conf->ssl_protocols, prev->ssl_protocols, (NGX_CONF_BITMASK_SET +#ifndef SSL_OP_NO_TLSv1_2 |NGX_SSL_TLSv1|NGX_SSL_TLSv1_1 +#endif |NGX_SSL_TLSv1_2|NGX_SSL_TLSv1_3)); ngx_conf_merge_str_value(conf->ssl_ciphers, prev->ssl_ciphers, diff --git a/src/http/modules/ngx_http_ssl_module.c b/src/http/modules/ngx_http_ssl_module.c index abc8d49ab69..1fb1e6129be 100644 --- a/src/http/modules/ngx_http_ssl_module.c +++ b/src/http/modules/ngx_http_ssl_module.c @@ -653,7 +653,9 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) ngx_conf_merge_bitmask_value(conf->protocols, prev->protocols, (NGX_CONF_BITMASK_SET +#ifndef SSL_OP_NO_TLSv1_2 |NGX_SSL_TLSv1|NGX_SSL_TLSv1_1 +#endif |NGX_SSL_TLSv1_2|NGX_SSL_TLSv1_3)); ngx_conf_merge_size_value(conf->buffer_size, prev->buffer_size, diff --git a/src/http/modules/ngx_http_uwsgi_module.c b/src/http/modules/ngx_http_uwsgi_module.c index c965d4bbd15..f2a8dbe6a08 100644 --- a/src/http/modules/ngx_http_uwsgi_module.c +++ b/src/http/modules/ngx_http_uwsgi_module.c @@ -1879,7 +1879,9 @@ ngx_http_uwsgi_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) ngx_conf_merge_bitmask_value(conf->ssl_protocols, prev->ssl_protocols, (NGX_CONF_BITMASK_SET +#ifndef SSL_OP_NO_TLSv1_2 |NGX_SSL_TLSv1|NGX_SSL_TLSv1_1 +#endif |NGX_SSL_TLSv1_2|NGX_SSL_TLSv1_3)); ngx_conf_merge_str_value(conf->ssl_ciphers, prev->ssl_ciphers, diff --git a/src/mail/ngx_mail_ssl_module.c b/src/mail/ngx_mail_ssl_module.c index b547dc1011a..2fee1adb8db 100644 --- a/src/mail/ngx_mail_ssl_module.c +++ b/src/mail/ngx_mail_ssl_module.c @@ -345,7 +345,9 @@ ngx_mail_ssl_merge_conf(ngx_conf_t *cf, void *parent, void *child) ngx_conf_merge_bitmask_value(conf->protocols, prev->protocols, (NGX_CONF_BITMASK_SET +#ifndef SSL_OP_NO_TLSv1_2 |NGX_SSL_TLSv1|NGX_SSL_TLSv1_1 +#endif |NGX_SSL_TLSv1_2|NGX_SSL_TLSv1_3)); ngx_conf_merge_uint_value(conf->verify, prev->verify, 0); diff --git a/src/stream/ngx_stream_proxy_module.c b/src/stream/ngx_stream_proxy_module.c index ed275c009a8..bbf4f7ec002 100644 --- a/src/stream/ngx_stream_proxy_module.c +++ b/src/stream/ngx_stream_proxy_module.c @@ -2164,7 +2164,9 @@ ngx_stream_proxy_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) ngx_conf_merge_bitmask_value(conf->ssl_protocols, prev->ssl_protocols, (NGX_CONF_BITMASK_SET +#ifndef SSL_OP_NO_TLSv1_2 |NGX_SSL_TLSv1|NGX_SSL_TLSv1_1 +#endif |NGX_SSL_TLSv1_2|NGX_SSL_TLSv1_3)); ngx_conf_merge_str_value(conf->ssl_ciphers, prev->ssl_ciphers, "DEFAULT"); diff --git a/src/stream/ngx_stream_ssl_module.c b/src/stream/ngx_stream_ssl_module.c index 0233a9258b7..55bc54a44c9 100644 --- a/src/stream/ngx_stream_ssl_module.c +++ b/src/stream/ngx_stream_ssl_module.c @@ -884,7 +884,9 @@ ngx_stream_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) ngx_conf_merge_bitmask_value(conf->protocols, prev->protocols, (NGX_CONF_BITMASK_SET +#ifndef SSL_OP_NO_TLSv1_2 |NGX_SSL_TLSv1|NGX_SSL_TLSv1_1 +#endif |NGX_SSL_TLSv1_2|NGX_SSL_TLSv1_3)); ngx_conf_merge_uint_value(conf->verify, prev->verify, 0); From db6870e06dde7ab249e9a41a0e0a76219f82dd8c Mon Sep 17 00:00:00 2001 From: Ruslan Ermilov Date: Sat, 15 Feb 2014 15:12:34 +0400 Subject: [PATCH 181/279] Upstream: re-resolvable servers. Specifying the upstream server by a hostname together with the "resolve" parameter will make the hostname to be periodically resolved, and upstream servers added/removed as necessary. This requires a "resolver" at the "http" configuration block. The "resolver_timeout" parameter also affects when the failed DNS requests will be attempted again. Responses with NXDOMAIN will be attempted again in 10 seconds. Upstream has a configuration generation number that is incremented each time servers are added/removed to the primary/backup list. This number is remembered by the peer.init method, and if peer.get detects a change in configuration, it returns NGX_BUSY. Each server has a reference counter. It is incremented by peer.get and decremented by peer.free. When a server is removed, it is removed from the list of servers and is marked as "zombie". The memory allocated by a zombie peer is freed only when its reference count becomes zero. Co-authored-by: Roman Arutyunyan Co-authored-by: Sergey Kandaurov Co-authored-by: Vladimir Homutov --- .../modules/ngx_http_upstream_hash_module.c | 88 ++++- .../ngx_http_upstream_ip_hash_module.c | 12 +- .../ngx_http_upstream_least_conn_module.c | 12 + .../modules/ngx_http_upstream_random_module.c | 35 +- .../modules/ngx_http_upstream_zone_module.c | 362 +++++++++++++++++- src/http/ngx_http_upstream.c | 78 ++++ src/http/ngx_http_upstream.h | 9 +- src/http/ngx_http_upstream_round_robin.c | 203 +++++++++- src/http/ngx_http_upstream_round_robin.h | 86 ++++- src/stream/ngx_stream_proxy_module.c | 19 + src/stream/ngx_stream_upstream.c | 58 +++ src/stream/ngx_stream_upstream.h | 9 +- src/stream/ngx_stream_upstream_hash_module.c | 90 ++++- .../ngx_stream_upstream_least_conn_module.c | 12 + .../ngx_stream_upstream_random_module.c | 35 +- src/stream/ngx_stream_upstream_round_robin.c | 204 +++++++++- src/stream/ngx_stream_upstream_round_robin.h | 86 ++++- src/stream/ngx_stream_upstream_zone_module.c | 362 +++++++++++++++++- 18 files changed, 1704 insertions(+), 56 deletions(-) diff --git a/src/http/modules/ngx_http_upstream_hash_module.c b/src/http/modules/ngx_http_upstream_hash_module.c index e741eb237a2..2ecc8d3fb3e 100644 --- a/src/http/modules/ngx_http_upstream_hash_module.c +++ b/src/http/modules/ngx_http_upstream_hash_module.c @@ -24,6 +24,9 @@ typedef struct { typedef struct { ngx_http_complex_value_t key; +#if (NGX_HTTP_UPSTREAM_ZONE) + ngx_uint_t config; +#endif ngx_http_upstream_chash_points_t *points; } ngx_http_upstream_hash_srv_conf_t; @@ -49,6 +52,8 @@ static ngx_int_t ngx_http_upstream_get_hash_peer(ngx_peer_connection_t *pc, static ngx_int_t ngx_http_upstream_init_chash(ngx_conf_t *cf, ngx_http_upstream_srv_conf_t *us); +static ngx_int_t ngx_http_upstream_update_chash(ngx_pool_t *pool, + ngx_http_upstream_srv_conf_t *us); static int ngx_libc_cdecl ngx_http_upstream_chash_cmp_points(const void *one, const void *two); static ngx_uint_t ngx_http_upstream_find_chash_point( @@ -178,10 +183,17 @@ ngx_http_upstream_get_hash_peer(ngx_peer_connection_t *pc, void *data) ngx_http_upstream_rr_peers_rlock(hp->rrp.peers); - if (hp->tries > 20 || hp->rrp.peers->single || hp->key.len == 0) { + if (hp->tries > 20 || hp->rrp.peers->number < 2 || hp->key.len == 0) { + ngx_http_upstream_rr_peers_unlock(hp->rrp.peers); + return hp->get_rr_peer(pc, &hp->rrp); + } + +#if (NGX_HTTP_UPSTREAM_ZONE) + if (hp->rrp.peers->config && hp->rrp.config != *hp->rrp.peers->config) { ngx_http_upstream_rr_peers_unlock(hp->rrp.peers); return hp->get_rr_peer(pc, &hp->rrp); } +#endif now = ngx_time(); @@ -262,6 +274,7 @@ ngx_http_upstream_get_hash_peer(ngx_peer_connection_t *pc, void *data) } hp->rrp.current = peer; + ngx_http_upstream_rr_peer_ref(hp->rrp.peers, peer); pc->sockaddr = peer->sockaddr; pc->socklen = peer->socklen; @@ -284,6 +297,26 @@ ngx_http_upstream_get_hash_peer(ngx_peer_connection_t *pc, void *data) static ngx_int_t ngx_http_upstream_init_chash(ngx_conf_t *cf, ngx_http_upstream_srv_conf_t *us) +{ + if (ngx_http_upstream_init_round_robin(cf, us) != NGX_OK) { + return NGX_ERROR; + } + + us->peer.init = ngx_http_upstream_init_chash_peer; + +#if (NGX_HTTP_UPSTREAM_ZONE) + if (us->shm_zone) { + return NGX_OK; + } +#endif + + return ngx_http_upstream_update_chash(cf->pool, us); +} + + +static ngx_int_t +ngx_http_upstream_update_chash(ngx_pool_t *pool, + ngx_http_upstream_srv_conf_t *us) { u_char *host, *port, c; size_t host_len, port_len, size; @@ -299,25 +332,32 @@ ngx_http_upstream_init_chash(ngx_conf_t *cf, ngx_http_upstream_srv_conf_t *us) u_char byte[4]; } prev_hash; - if (ngx_http_upstream_init_round_robin(cf, us) != NGX_OK) { - return NGX_ERROR; - } + hcf = ngx_http_conf_upstream_srv_conf(us, ngx_http_upstream_hash_module); - us->peer.init = ngx_http_upstream_init_chash_peer; + if (hcf->points) { + ngx_free(hcf->points); + hcf->points = NULL; + } peers = us->peer.data; npoints = peers->total_weight * 160; size = sizeof(ngx_http_upstream_chash_points_t) - + sizeof(ngx_http_upstream_chash_point_t) * (npoints - 1); + - sizeof(ngx_http_upstream_chash_point_t) + + sizeof(ngx_http_upstream_chash_point_t) * npoints; - points = ngx_palloc(cf->pool, size); + points = pool ? ngx_palloc(pool, size) : ngx_alloc(size, ngx_cycle->log); if (points == NULL) { return NGX_ERROR; } points->number = 0; + if (npoints == 0) { + hcf->points = points; + return NGX_OK; + } + for (peer = peers->peer; peer; peer = peer->next) { server = &peer->server; @@ -401,7 +441,6 @@ ngx_http_upstream_init_chash(ngx_conf_t *cf, ngx_http_upstream_srv_conf_t *us) points->number = i + 1; - hcf = ngx_http_conf_upstream_srv_conf(us, ngx_http_upstream_hash_module); hcf->points = points; return NGX_OK; @@ -481,7 +520,22 @@ ngx_http_upstream_init_chash_peer(ngx_http_request_t *r, ngx_http_upstream_rr_peers_rlock(hp->rrp.peers); - hp->hash = ngx_http_upstream_find_chash_point(hcf->points, hash); +#if (NGX_HTTP_UPSTREAM_ZONE) + if (hp->rrp.peers->config + && (hcf->points == NULL || hcf->config != *hp->rrp.peers->config)) + { + if (ngx_http_upstream_update_chash(NULL, us) != NGX_OK) { + ngx_http_upstream_rr_peers_unlock(hp->rrp.peers); + return NGX_ERROR; + } + + hcf->config = *hp->rrp.peers->config; + } +#endif + + if (hcf->points->number) { + hp->hash = ngx_http_upstream_find_chash_point(hcf->points, hash); + } ngx_http_upstream_rr_peers_unlock(hp->rrp.peers); @@ -517,6 +571,20 @@ ngx_http_upstream_get_chash_peer(ngx_peer_connection_t *pc, void *data) pc->cached = 0; pc->connection = NULL; + if (hp->rrp.peers->number == 0) { + pc->name = hp->rrp.peers->name; + ngx_http_upstream_rr_peers_unlock(hp->rrp.peers); + return NGX_BUSY; + } + +#if (NGX_HTTP_UPSTREAM_ZONE) + if (hp->rrp.peers->config && hp->rrp.config != *hp->rrp.peers->config) { + pc->name = hp->rrp.peers->name; + ngx_http_upstream_rr_peers_unlock(hp->rrp.peers); + return NGX_BUSY; + } +#endif + now = ngx_time(); hcf = hp->conf; @@ -597,6 +665,7 @@ ngx_http_upstream_get_chash_peer(ngx_peer_connection_t *pc, void *data) found: hp->rrp.current = best; + ngx_http_upstream_rr_peer_ref(hp->rrp.peers, best); pc->sockaddr = best->sockaddr; pc->socklen = best->socklen; @@ -664,6 +733,7 @@ ngx_http_upstream_hash(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) } uscf->flags = NGX_HTTP_UPSTREAM_CREATE + |NGX_HTTP_UPSTREAM_MODIFY |NGX_HTTP_UPSTREAM_WEIGHT |NGX_HTTP_UPSTREAM_MAX_CONNS |NGX_HTTP_UPSTREAM_MAX_FAILS diff --git a/src/http/modules/ngx_http_upstream_ip_hash_module.c b/src/http/modules/ngx_http_upstream_ip_hash_module.c index 1fa01d95aff..1c1b41d1920 100644 --- a/src/http/modules/ngx_http_upstream_ip_hash_module.c +++ b/src/http/modules/ngx_http_upstream_ip_hash_module.c @@ -163,11 +163,19 @@ ngx_http_upstream_get_ip_hash_peer(ngx_peer_connection_t *pc, void *data) ngx_http_upstream_rr_peers_rlock(iphp->rrp.peers); - if (iphp->tries > 20 || iphp->rrp.peers->single) { + if (iphp->tries > 20 || iphp->rrp.peers->number < 2) { ngx_http_upstream_rr_peers_unlock(iphp->rrp.peers); return iphp->get_rr_peer(pc, &iphp->rrp); } +#if (NGX_HTTP_UPSTREAM_ZONE) + if (iphp->rrp.peers->config && iphp->rrp.config != *iphp->rrp.peers->config) + { + ngx_http_upstream_rr_peers_unlock(iphp->rrp.peers); + return iphp->get_rr_peer(pc, &iphp->rrp); + } +#endif + now = ngx_time(); pc->cached = 0; @@ -232,6 +240,7 @@ ngx_http_upstream_get_ip_hash_peer(ngx_peer_connection_t *pc, void *data) } iphp->rrp.current = peer; + ngx_http_upstream_rr_peer_ref(iphp->rrp.peers, peer); pc->sockaddr = peer->sockaddr; pc->socklen = peer->socklen; @@ -268,6 +277,7 @@ ngx_http_upstream_ip_hash(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) uscf->peer.init_upstream = ngx_http_upstream_init_ip_hash; uscf->flags = NGX_HTTP_UPSTREAM_CREATE + |NGX_HTTP_UPSTREAM_MODIFY |NGX_HTTP_UPSTREAM_WEIGHT |NGX_HTTP_UPSTREAM_MAX_CONNS |NGX_HTTP_UPSTREAM_MAX_FAILS diff --git a/src/http/modules/ngx_http_upstream_least_conn_module.c b/src/http/modules/ngx_http_upstream_least_conn_module.c index ebe06276d57..4df97777c76 100644 --- a/src/http/modules/ngx_http_upstream_least_conn_module.c +++ b/src/http/modules/ngx_http_upstream_least_conn_module.c @@ -124,6 +124,12 @@ ngx_http_upstream_get_least_conn_peer(ngx_peer_connection_t *pc, void *data) ngx_http_upstream_rr_peers_wlock(peers); +#if (NGX_HTTP_UPSTREAM_ZONE) + if (peers->config && rrp->config != *peers->config) { + goto busy; + } +#endif + best = NULL; total = 0; @@ -244,6 +250,7 @@ ngx_http_upstream_get_least_conn_peer(ngx_peer_connection_t *pc, void *data) best->conns++; rrp->current = best; + ngx_http_upstream_rr_peer_ref(peers, best); n = p / (8 * sizeof(uintptr_t)); m = (uintptr_t) 1 << p % (8 * sizeof(uintptr_t)); @@ -280,6 +287,10 @@ ngx_http_upstream_get_least_conn_peer(ngx_peer_connection_t *pc, void *data) ngx_http_upstream_rr_peers_wlock(peers); } +#if (NGX_HTTP_UPSTREAM_ZONE) +busy: +#endif + ngx_http_upstream_rr_peers_unlock(peers); pc->name = peers->name; @@ -303,6 +314,7 @@ ngx_http_upstream_least_conn(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) uscf->peer.init_upstream = ngx_http_upstream_init_least_conn; uscf->flags = NGX_HTTP_UPSTREAM_CREATE + |NGX_HTTP_UPSTREAM_MODIFY |NGX_HTTP_UPSTREAM_WEIGHT |NGX_HTTP_UPSTREAM_MAX_CONNS |NGX_HTTP_UPSTREAM_MAX_FAILS diff --git a/src/http/modules/ngx_http_upstream_random_module.c b/src/http/modules/ngx_http_upstream_random_module.c index 6f169c8f6ab..74714874b37 100644 --- a/src/http/modules/ngx_http_upstream_random_module.c +++ b/src/http/modules/ngx_http_upstream_random_module.c @@ -17,6 +17,9 @@ typedef struct { typedef struct { ngx_uint_t two; +#if (NGX_HTTP_UPSTREAM_ZONE) + ngx_uint_t config; +#endif ngx_http_upstream_random_range_t *ranges; } ngx_http_upstream_random_srv_conf_t; @@ -127,6 +130,11 @@ ngx_http_upstream_update_random(ngx_pool_t *pool, rcf = ngx_http_conf_upstream_srv_conf(us, ngx_http_upstream_random_module); + if (rcf->ranges) { + ngx_free(rcf->ranges); + rcf->ranges = NULL; + } + peers = us->peer.data; size = peers->number * sizeof(ngx_http_upstream_random_range_t); @@ -186,11 +194,15 @@ ngx_http_upstream_init_random_peer(ngx_http_request_t *r, ngx_http_upstream_rr_peers_rlock(rp->rrp.peers); #if (NGX_HTTP_UPSTREAM_ZONE) - if (rp->rrp.peers->shpool && rcf->ranges == NULL) { + if (rp->rrp.peers->config + && (rcf->ranges == NULL || rcf->config != *rp->rrp.peers->config)) + { if (ngx_http_upstream_update_random(NULL, us) != NGX_OK) { ngx_http_upstream_rr_peers_unlock(rp->rrp.peers); return NGX_ERROR; } + + rcf->config = *rp->rrp.peers->config; } #endif @@ -220,11 +232,18 @@ ngx_http_upstream_get_random_peer(ngx_peer_connection_t *pc, void *data) ngx_http_upstream_rr_peers_rlock(peers); - if (rp->tries > 20 || peers->single) { + if (rp->tries > 20 || peers->number < 2) { ngx_http_upstream_rr_peers_unlock(peers); return ngx_http_upstream_get_round_robin_peer(pc, rrp); } +#if (NGX_HTTP_UPSTREAM_ZONE) + if (peers->config && rrp->config != *peers->config) { + ngx_http_upstream_rr_peers_unlock(peers); + return ngx_http_upstream_get_round_robin_peer(pc, rrp); + } +#endif + pc->cached = 0; pc->connection = NULL; @@ -274,6 +293,7 @@ ngx_http_upstream_get_random_peer(ngx_peer_connection_t *pc, void *data) } rrp->current = peer; + ngx_http_upstream_rr_peer_ref(peers, peer); if (now - peer->checked > peer->fail_timeout) { peer->checked = now; @@ -314,10 +334,17 @@ ngx_http_upstream_get_random2_peer(ngx_peer_connection_t *pc, void *data) ngx_http_upstream_rr_peers_wlock(peers); - if (rp->tries > 20 || peers->single) { + if (rp->tries > 20 || peers->number < 2) { + ngx_http_upstream_rr_peers_unlock(peers); + return ngx_http_upstream_get_round_robin_peer(pc, rrp); + } + +#if (NGX_HTTP_UPSTREAM_ZONE) + if (peers->config && rrp->config != *peers->config) { ngx_http_upstream_rr_peers_unlock(peers); return ngx_http_upstream_get_round_robin_peer(pc, rrp); } +#endif pc->cached = 0; pc->connection = NULL; @@ -384,6 +411,7 @@ ngx_http_upstream_get_random2_peer(ngx_peer_connection_t *pc, void *data) } rrp->current = peer; + ngx_http_upstream_rr_peer_ref(peers, peer); if (now - peer->checked > peer->fail_timeout) { peer->checked = now; @@ -467,6 +495,7 @@ ngx_http_upstream_random(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) uscf->peer.init_upstream = ngx_http_upstream_init_random; uscf->flags = NGX_HTTP_UPSTREAM_CREATE + |NGX_HTTP_UPSTREAM_MODIFY |NGX_HTTP_UPSTREAM_WEIGHT |NGX_HTTP_UPSTREAM_MAX_CONNS |NGX_HTTP_UPSTREAM_MAX_FAILS diff --git a/src/http/modules/ngx_http_upstream_zone_module.c b/src/http/modules/ngx_http_upstream_zone_module.c index 3229cfe84a4..355df39abc8 100644 --- a/src/http/modules/ngx_http_upstream_zone_module.c +++ b/src/http/modules/ngx_http_upstream_zone_module.c @@ -18,6 +18,13 @@ static ngx_http_upstream_rr_peers_t *ngx_http_upstream_zone_copy_peers( ngx_slab_pool_t *shpool, ngx_http_upstream_srv_conf_t *uscf); static ngx_http_upstream_rr_peer_t *ngx_http_upstream_zone_copy_peer( ngx_http_upstream_rr_peers_t *peers, ngx_http_upstream_rr_peer_t *src); +static void ngx_http_upstream_zone_set_single( + ngx_http_upstream_srv_conf_t *uscf); +static void ngx_http_upstream_zone_remove_peer_locked( + ngx_http_upstream_rr_peers_t *peers, ngx_http_upstream_rr_peer_t *peer); +static ngx_int_t ngx_http_upstream_zone_init_worker(ngx_cycle_t *cycle); +static void ngx_http_upstream_zone_resolve_timer(ngx_event_t *event); +static void ngx_http_upstream_zone_resolve_handler(ngx_resolver_ctx_t *ctx); static ngx_command_t ngx_http_upstream_zone_commands[] = { @@ -55,7 +62,7 @@ ngx_module_t ngx_http_upstream_zone_module = { NGX_HTTP_MODULE, /* module type */ NULL, /* init master */ NULL, /* init module */ - NULL, /* init process */ + ngx_http_upstream_zone_init_worker, /* init process */ NULL, /* init thread */ NULL, /* exit thread */ NULL, /* exit process */ @@ -188,9 +195,15 @@ ngx_http_upstream_zone_copy_peers(ngx_slab_pool_t *shpool, ngx_http_upstream_srv_conf_t *uscf) { ngx_str_t *name; + ngx_uint_t *config; ngx_http_upstream_rr_peer_t *peer, **peerp; ngx_http_upstream_rr_peers_t *peers, *backup; + config = ngx_slab_calloc(shpool, sizeof(ngx_uint_t)); + if (config == NULL) { + return NULL; + } + peers = ngx_slab_alloc(shpool, sizeof(ngx_http_upstream_rr_peers_t)); if (peers == NULL) { return NULL; @@ -214,6 +227,7 @@ ngx_http_upstream_zone_copy_peers(ngx_slab_pool_t *shpool, peers->name = name; peers->shpool = shpool; + peers->config = config; for (peerp = &peers->peer; *peerp; peerp = &peer->next) { /* pool is unlocked */ @@ -223,6 +237,17 @@ ngx_http_upstream_zone_copy_peers(ngx_slab_pool_t *shpool, } *peerp = peer; + (*peers->config)++; + } + + for (peerp = &peers->resolve; *peerp; peerp = &peer->next) { + peer = ngx_http_upstream_zone_copy_peer(peers, *peerp); + if (peer == NULL) { + return NULL; + } + + *peerp = peer; + (*peers->config)++; } if (peers->next == NULL) { @@ -239,6 +264,7 @@ ngx_http_upstream_zone_copy_peers(ngx_slab_pool_t *shpool, backup->name = name; backup->shpool = shpool; + backup->config = config; for (peerp = &backup->peer; *peerp; peerp = &peer->next) { /* pool is unlocked */ @@ -248,6 +274,17 @@ ngx_http_upstream_zone_copy_peers(ngx_slab_pool_t *shpool, } *peerp = peer; + (*backup->config)++; + } + + for (peerp = &backup->resolve; *peerp; peerp = &peer->next) { + peer = ngx_http_upstream_zone_copy_peer(backup, *peerp); + if (peer == NULL) { + return NULL; + } + + *peerp = peer; + (*backup->config)++; } peers->next = backup; @@ -279,6 +316,7 @@ ngx_http_upstream_zone_copy_peer(ngx_http_upstream_rr_peers_t *peers, dst->sockaddr = NULL; dst->name.data = NULL; dst->server.data = NULL; + dst->host = NULL; } dst->sockaddr = ngx_slab_calloc_locked(pool, sizeof(ngx_sockaddr_t)); @@ -301,12 +339,37 @@ ngx_http_upstream_zone_copy_peer(ngx_http_upstream_rr_peers_t *peers, } ngx_memcpy(dst->server.data, src->server.data, src->server.len); + + if (src->host) { + dst->host = ngx_slab_calloc_locked(pool, + sizeof(ngx_http_upstream_host_t)); + if (dst->host == NULL) { + goto failed; + } + + dst->host->name.data = ngx_slab_alloc_locked(pool, + src->host->name.len); + if (dst->host->name.data == NULL) { + goto failed; + } + + dst->host->peers = peers; + dst->host->peer = dst; + + dst->host->name.len = src->host->name.len; + ngx_memcpy(dst->host->name.data, src->host->name.data, + src->host->name.len); + } } return dst; failed: + if (dst->host) { + ngx_slab_free_locked(pool, dst->host); + } + if (dst->server.data) { ngx_slab_free_locked(pool, dst->server.data); } @@ -323,3 +386,300 @@ ngx_http_upstream_zone_copy_peer(ngx_http_upstream_rr_peers_t *peers, return NULL; } + + +static void +ngx_http_upstream_zone_set_single(ngx_http_upstream_srv_conf_t *uscf) +{ + ngx_http_upstream_rr_peers_t *peers; + + peers = uscf->peer.data; + + if (peers->number == 1 + && (peers->next == NULL || peers->next->number == 0)) + { + peers->single = 1; + + } else { + peers->single = 0; + } +} + + +static void +ngx_http_upstream_zone_remove_peer_locked(ngx_http_upstream_rr_peers_t *peers, + ngx_http_upstream_rr_peer_t *peer) +{ + peers->total_weight -= peer->weight; + peers->number--; + peers->tries -= (peer->down == 0); + (*peers->config)++; + peers->weighted = (peers->total_weight != peers->number); + + ngx_http_upstream_rr_peer_free(peers, peer); +} + + +static ngx_int_t +ngx_http_upstream_zone_init_worker(ngx_cycle_t *cycle) +{ + ngx_uint_t i; + ngx_event_t *event; + ngx_http_upstream_rr_peer_t *peer; + ngx_http_upstream_rr_peers_t *peers; + ngx_http_upstream_srv_conf_t *uscf, **uscfp; + ngx_http_upstream_main_conf_t *umcf; + + if (ngx_process != NGX_PROCESS_WORKER + && ngx_process != NGX_PROCESS_SINGLE) + { + return NGX_OK; + } + + umcf = ngx_http_cycle_get_module_main_conf(cycle, ngx_http_upstream_module); + + if (umcf == NULL) { + return NGX_OK; + } + + uscfp = umcf->upstreams.elts; + + for (i = 0; i < umcf->upstreams.nelts; i++) { + + uscf = uscfp[i]; + + if (uscf->shm_zone == NULL) { + continue; + } + + peers = uscf->peer.data; + + do { + ngx_http_upstream_rr_peers_wlock(peers); + + for (peer = peers->resolve; peer; peer = peer->next) { + + if (peer->host->worker != ngx_worker) { + continue; + } + + event = &peer->host->event; + ngx_memzero(event, sizeof(ngx_event_t)); + + event->data = uscf; + event->handler = ngx_http_upstream_zone_resolve_timer; + event->log = cycle->log; + event->cancelable = 1; + + ngx_add_timer(event, 1); + } + + ngx_http_upstream_rr_peers_unlock(peers); + + peers = peers->next; + + } while (peers); + } + + return NGX_OK; +} + + +static void +ngx_http_upstream_zone_resolve_timer(ngx_event_t *event) +{ + ngx_resolver_ctx_t *ctx; + ngx_http_upstream_host_t *host; + ngx_http_upstream_srv_conf_t *uscf; + + host = (ngx_http_upstream_host_t *) event; + uscf = event->data; + + ctx = ngx_resolve_start(uscf->resolver, NULL); + if (ctx == NULL) { + goto retry; + } + + if (ctx == NGX_NO_RESOLVER) { + ngx_log_error(NGX_LOG_ERR, event->log, 0, + "no resolver defined to resolve %V", &host->name); + return; + } + + ctx->name = host->name; + ctx->handler = ngx_http_upstream_zone_resolve_handler; + ctx->data = host; + ctx->timeout = uscf->resolver_timeout; + ctx->cancelable = 1; + + if (ngx_resolve_name(ctx) == NGX_OK) { + return; + } + +retry: + + ngx_add_timer(event, ngx_max(uscf->resolver_timeout, 1000)); +} + + +static void +ngx_http_upstream_zone_resolve_handler(ngx_resolver_ctx_t *ctx) +{ + time_t now; + in_port_t port; + ngx_msec_t timer; + ngx_uint_t i, j; + ngx_event_t *event; + ngx_resolver_addr_t *addr; + ngx_http_upstream_host_t *host; + ngx_http_upstream_rr_peer_t *peer, *template, **peerp; + ngx_http_upstream_rr_peers_t *peers; + ngx_http_upstream_srv_conf_t *uscf; + + host = ctx->data; + event = &host->event; + uscf = event->data; + peers = host->peers; + template = host->peer; + + ngx_http_upstream_rr_peers_wlock(peers); + + now = ngx_time(); + + if (ctx->state) { + ngx_log_error(NGX_LOG_ERR, event->log, 0, + "%V could not be resolved (%i: %s)", + &ctx->name, ctx->state, + ngx_resolver_strerror(ctx->state)); + + if (ctx->state != NGX_RESOLVE_NXDOMAIN) { + ngx_http_upstream_rr_peers_unlock(peers); + + ngx_resolve_name_done(ctx); + + ngx_add_timer(event, ngx_max(uscf->resolver_timeout, 1000)); + return; + } + + /* NGX_RESOLVE_NXDOMAIN */ + + ctx->naddrs = 0; + } + +#if (NGX_DEBUG) + { + u_char text[NGX_SOCKADDR_STRLEN]; + size_t len; + + for (i = 0; i < ctx->naddrs; i++) { + len = ngx_sock_ntop(ctx->addrs[i].sockaddr, ctx->addrs[i].socklen, + text, NGX_SOCKADDR_STRLEN, 0); + + ngx_log_debug3(NGX_LOG_DEBUG_HTTP, event->log, 0, + "name %V was resolved to %*s", &host->name, len, text); + } + } +#endif + + for (peerp = &peers->peer; *peerp; /* void */ ) { + peer = *peerp; + + if (peer->host != host) { + goto next; + } + + for (j = 0; j < ctx->naddrs; j++) { + + addr = &ctx->addrs[j]; + + if (addr->name.len == 0 + && ngx_cmp_sockaddr(peer->sockaddr, peer->socklen, + addr->sockaddr, addr->socklen, 0) + == NGX_OK) + { + addr->name.len = 1; + goto next; + } + } + + *peerp = peer->next; + ngx_http_upstream_zone_remove_peer_locked(peers, peer); + + ngx_http_upstream_zone_set_single(uscf); + + continue; + + next: + + peerp = &peer->next; + } + + for (i = 0; i < ctx->naddrs; i++) { + + addr = &ctx->addrs[i]; + + if (addr->name.len == 1) { + addr->name.len = 0; + continue; + } + + ngx_shmtx_lock(&peers->shpool->mutex); + peer = ngx_http_upstream_zone_copy_peer(peers, NULL); + ngx_shmtx_unlock(&peers->shpool->mutex); + + if (peer == NULL) { + ngx_log_error(NGX_LOG_ERR, event->log, 0, + "cannot add new server to upstream \"%V\", " + "memory exhausted", peers->name); + break; + } + + ngx_memcpy(peer->sockaddr, addr->sockaddr, addr->socklen); + + port = ((struct sockaddr_in *) template->sockaddr)->sin_port; + + switch (peer->sockaddr->sa_family) { +#if (NGX_HAVE_INET6) + case AF_INET6: + ((struct sockaddr_in6 *) peer->sockaddr)->sin6_port = port; + break; +#endif + default: /* AF_INET */ + ((struct sockaddr_in *) peer->sockaddr)->sin_port = port; + } + + peer->socklen = addr->socklen; + + peer->name.len = ngx_sock_ntop(peer->sockaddr, peer->socklen, + peer->name.data, NGX_SOCKADDR_STRLEN, 1); + + peer->host = template->host; + peer->server = template->server; + + peer->weight = template->weight; + peer->effective_weight = peer->weight; + peer->max_conns = template->max_conns; + peer->max_fails = template->max_fails; + peer->fail_timeout = template->fail_timeout; + peer->down = template->down; + + *peerp = peer; + peerp = &peer->next; + + peers->number++; + peers->tries += (peer->down == 0); + peers->total_weight += peer->weight; + peers->weighted = (peers->total_weight != peers->number); + (*peers->config)++; + + ngx_http_upstream_zone_set_single(uscf); + } + + ngx_http_upstream_rr_peers_unlock(peers); + + timer = (ngx_msec_t) 1000 * (ctx->valid > now ? ctx->valid - now + 1 : 1); + + ngx_resolve_name_done(ctx); + + ngx_add_timer(event, timer); +} diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c index d29e503cb78..efe77243913 100644 --- a/src/http/ngx_http_upstream.c +++ b/src/http/ngx_http_upstream.c @@ -1565,6 +1565,26 @@ ngx_http_upstream_connect(ngx_http_request_t *r, ngx_http_upstream_t *u) u->state->peer = u->peer.name; +#if (NGX_HTTP_UPSTREAM_ZONE) + if (u->upstream && u->upstream->shm_zone + && (u->upstream->flags & NGX_HTTP_UPSTREAM_MODIFY)) + { + u->state->peer = ngx_palloc(r->pool, + sizeof(ngx_str_t) + u->peer.name->len); + if (u->state->peer == NULL) { + ngx_http_upstream_finalize_request(r, u, + NGX_HTTP_INTERNAL_SERVER_ERROR); + return; + } + + u->state->peer->len = u->peer.name->len; + u->state->peer->data = (u_char *) (u->state->peer + 1); + ngx_memcpy(u->state->peer->data, u->peer.name->data, u->peer.name->len); + + u->peer.name = u->state->peer; + } +#endif + if (rc == NGX_BUSY) { ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, "no live upstreams"); ngx_http_upstream_next(r, u, NGX_HTTP_UPSTREAM_FT_NOLIVE); @@ -6066,6 +6086,7 @@ ngx_http_upstream(ngx_conf_t *cf, ngx_command_t *cmd, void *dummy) u.no_port = 1; uscf = ngx_http_upstream_add(cf, &u, NGX_HTTP_UPSTREAM_CREATE + |NGX_HTTP_UPSTREAM_MODIFY |NGX_HTTP_UPSTREAM_WEIGHT |NGX_HTTP_UPSTREAM_MAX_CONNS |NGX_HTTP_UPSTREAM_MAX_FAILS @@ -6171,6 +6192,9 @@ ngx_http_upstream_server(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) ngx_url_t u; ngx_int_t weight, max_conns, max_fails; ngx_uint_t i; +#if (NGX_HTTP_UPSTREAM_ZONE) + ngx_uint_t resolve; +#endif ngx_http_upstream_server_t *us; us = ngx_array_push(uscf->servers); @@ -6186,6 +6210,9 @@ ngx_http_upstream_server(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) max_conns = 0; max_fails = 1; fail_timeout = 10; +#if (NGX_HTTP_UPSTREAM_ZONE) + resolve = 0; +#endif for (i = 2; i < cf->args->nelts; i++) { @@ -6274,6 +6301,13 @@ ngx_http_upstream_server(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) continue; } +#if (NGX_HTTP_UPSTREAM_ZONE) + if (ngx_strcmp(value[i].data, "resolve") == 0) { + resolve = 1; + continue; + } +#endif + goto invalid; } @@ -6282,6 +6316,13 @@ ngx_http_upstream_server(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) u.url = value[1]; u.default_port = 80; +#if (NGX_HTTP_UPSTREAM_ZONE) + if (resolve) { + /* resolve at run time */ + u.no_resolve = 1; + } +#endif + if (ngx_parse_url(cf->pool, &u) != NGX_OK) { if (u.err) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, @@ -6292,8 +6333,45 @@ ngx_http_upstream_server(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) } us->name = u.url; + +#if (NGX_HTTP_UPSTREAM_ZONE) + + if (resolve && u.naddrs == 0) { + ngx_addr_t *addr; + + /* save port */ + + addr = ngx_pcalloc(cf->pool, sizeof(ngx_addr_t)); + if (addr == NULL) { + return NGX_CONF_ERROR; + } + + addr->sockaddr = ngx_palloc(cf->pool, u.socklen); + if (addr->sockaddr == NULL) { + return NGX_CONF_ERROR; + } + + ngx_memcpy(addr->sockaddr, &u.sockaddr, u.socklen); + + addr->socklen = u.socklen; + + us->addrs = addr; + us->naddrs = 1; + + us->host = u.host; + + } else { + us->addrs = u.addrs; + us->naddrs = u.naddrs; + } + +#else + us->addrs = u.addrs; us->naddrs = u.naddrs; + +#endif + us->weight = weight; us->max_conns = max_conns; us->max_fails = max_fails; diff --git a/src/http/ngx_http_upstream.h b/src/http/ngx_http_upstream.h index db0e1424e11..0922021bde2 100644 --- a/src/http/ngx_http_upstream.h +++ b/src/http/ngx_http_upstream.h @@ -104,7 +104,11 @@ typedef struct { unsigned backup:1; - NGX_COMPAT_BEGIN(6) +#if (NGX_HTTP_UPSTREAM_ZONE) + ngx_str_t host; +#endif + + NGX_COMPAT_BEGIN(4) NGX_COMPAT_END } ngx_http_upstream_server_t; @@ -115,6 +119,7 @@ typedef struct { #define NGX_HTTP_UPSTREAM_FAIL_TIMEOUT 0x0008 #define NGX_HTTP_UPSTREAM_DOWN 0x0010 #define NGX_HTTP_UPSTREAM_BACKUP 0x0020 +#define NGX_HTTP_UPSTREAM_MODIFY 0x0040 #define NGX_HTTP_UPSTREAM_MAX_CONNS 0x0100 @@ -133,6 +138,8 @@ struct ngx_http_upstream_srv_conf_s { #if (NGX_HTTP_UPSTREAM_ZONE) ngx_shm_zone_t *shm_zone; + ngx_resolver_t *resolver; + ngx_msec_t resolver_timeout; #endif }; diff --git a/src/http/ngx_http_upstream_round_robin.c b/src/http/ngx_http_upstream_round_robin.c index 1f15fae502f..cc1b6d1a2e5 100644 --- a/src/http/ngx_http_upstream_round_robin.c +++ b/src/http/ngx_http_upstream_round_robin.c @@ -32,10 +32,15 @@ ngx_http_upstream_init_round_robin(ngx_conf_t *cf, ngx_http_upstream_srv_conf_t *us) { ngx_url_t u; - ngx_uint_t i, j, n, w, t; + ngx_uint_t i, j, n, r, w, t; ngx_http_upstream_server_t *server; ngx_http_upstream_rr_peer_t *peer, **peerp; ngx_http_upstream_rr_peers_t *peers, *backup; +#if (NGX_HTTP_UPSTREAM_ZONE) + ngx_uint_t resolve; + ngx_http_core_loc_conf_t *clcf; + ngx_http_upstream_rr_peer_t **rpeerp; +#endif us->peer.init = ngx_http_upstream_init_round_robin_peer; @@ -43,14 +48,33 @@ ngx_http_upstream_init_round_robin(ngx_conf_t *cf, server = us->servers->elts; n = 0; + r = 0; w = 0; t = 0; +#if (NGX_HTTP_UPSTREAM_ZONE) + resolve = 0; +#endif + for (i = 0; i < us->servers->nelts; i++) { + +#if (NGX_HTTP_UPSTREAM_ZONE) + if (server[i].host.len) { + resolve = 1; + } +#endif + if (server[i].backup) { continue; } +#if (NGX_HTTP_UPSTREAM_ZONE) + if (server[i].host.len) { + r++; + continue; + } +#endif + n += server[i].naddrs; w += server[i].naddrs * server[i].weight; @@ -59,7 +83,53 @@ ngx_http_upstream_init_round_robin(ngx_conf_t *cf, } } - if (n == 0) { +#if (NGX_HTTP_UPSTREAM_ZONE) + if (us->shm_zone) { + + if (resolve && !(us->flags & NGX_HTTP_UPSTREAM_MODIFY)) { + ngx_log_error(NGX_LOG_EMERG, cf->log, 0, + "load balancing method does not support" + " resolving names at run time in" + " upstream \"%V\" in %s:%ui", + &us->host, us->file_name, us->line); + return NGX_ERROR; + } + + clcf = ngx_http_conf_get_module_loc_conf(cf, ngx_http_core_module); + + us->resolver = clcf->resolver; + us->resolver_timeout = clcf->resolver_timeout; + + /* + * Without "resolver_timeout" in http{}, the value is unset. + * Even if we set it in ngx_http_core_merge_loc_conf(), it's + * still dependent on the module order and unreliable. + */ + ngx_conf_init_msec_value(us->resolver_timeout, 30000); + + if (resolve + && (us->resolver == NULL + || us->resolver->connections.nelts == 0)) + { + ngx_log_error(NGX_LOG_EMERG, cf->log, 0, + "no resolver defined to resolve names" + " at run time in upstream \"%V\" in %s:%ui", + &us->host, us->file_name, us->line); + return NGX_ERROR; + } + + } else if (resolve) { + + ngx_log_error(NGX_LOG_EMERG, cf->log, 0, + "resolving names at run time requires" + " upstream \"%V\" in %s:%ui" + " to be in shared memory", + &us->host, us->file_name, us->line); + return NGX_ERROR; + } +#endif + + if (n + r == 0) { ngx_log_error(NGX_LOG_EMERG, cf->log, 0, "no servers in upstream \"%V\" in %s:%ui", &us->host, us->file_name, us->line); @@ -71,7 +141,8 @@ ngx_http_upstream_init_round_robin(ngx_conf_t *cf, return NGX_ERROR; } - peer = ngx_pcalloc(cf->pool, sizeof(ngx_http_upstream_rr_peer_t) * n); + peer = ngx_pcalloc(cf->pool, sizeof(ngx_http_upstream_rr_peer_t) + * (n + r)); if (peer == NULL) { return NGX_ERROR; } @@ -86,11 +157,46 @@ ngx_http_upstream_init_round_robin(ngx_conf_t *cf, n = 0; peerp = &peers->peer; +#if (NGX_HTTP_UPSTREAM_ZONE) + rpeerp = &peers->resolve; +#endif + for (i = 0; i < us->servers->nelts; i++) { if (server[i].backup) { continue; } +#if (NGX_HTTP_UPSTREAM_ZONE) + if (server[i].host.len) { + + peer[n].host = ngx_pcalloc(cf->pool, + sizeof(ngx_http_upstream_host_t)); + if (peer[n].host == NULL) { + return NGX_ERROR; + } + + peer[n].host->name = server[i].host; + + peer[n].sockaddr = server[i].addrs[0].sockaddr; + peer[n].socklen = server[i].addrs[0].socklen; + peer[n].name = server[i].addrs[0].name; + peer[n].weight = server[i].weight; + peer[n].effective_weight = server[i].weight; + peer[n].current_weight = 0; + peer[n].max_conns = server[i].max_conns; + peer[n].max_fails = server[i].max_fails; + peer[n].fail_timeout = server[i].fail_timeout; + peer[n].down = server[i].down; + peer[n].server = server[i].name; + + *rpeerp = &peer[n]; + rpeerp = &peer[n].next; + n++; + + continue; + } +#endif + for (j = 0; j < server[i].naddrs; j++) { peer[n].sockaddr = server[i].addrs[j].sockaddr; peer[n].socklen = server[i].addrs[j].socklen; @@ -115,6 +221,7 @@ ngx_http_upstream_init_round_robin(ngx_conf_t *cf, /* backup servers */ n = 0; + r = 0; w = 0; t = 0; @@ -123,6 +230,13 @@ ngx_http_upstream_init_round_robin(ngx_conf_t *cf, continue; } +#if (NGX_HTTP_UPSTREAM_ZONE) + if (server[i].host.len) { + r++; + continue; + } +#endif + n += server[i].naddrs; w += server[i].naddrs * server[i].weight; @@ -131,7 +245,7 @@ ngx_http_upstream_init_round_robin(ngx_conf_t *cf, } } - if (n == 0) { + if (n + r == 0) { return NGX_OK; } @@ -140,12 +254,16 @@ ngx_http_upstream_init_round_robin(ngx_conf_t *cf, return NGX_ERROR; } - peer = ngx_pcalloc(cf->pool, sizeof(ngx_http_upstream_rr_peer_t) * n); + peer = ngx_pcalloc(cf->pool, sizeof(ngx_http_upstream_rr_peer_t) + * (n + r)); if (peer == NULL) { return NGX_ERROR; } - peers->single = 0; + if (n > 0) { + peers->single = 0; + } + backup->single = 0; backup->number = n; backup->weighted = (w != n); @@ -156,11 +274,46 @@ ngx_http_upstream_init_round_robin(ngx_conf_t *cf, n = 0; peerp = &backup->peer; +#if (NGX_HTTP_UPSTREAM_ZONE) + rpeerp = &backup->resolve; +#endif + for (i = 0; i < us->servers->nelts; i++) { if (!server[i].backup) { continue; } +#if (NGX_HTTP_UPSTREAM_ZONE) + if (server[i].host.len) { + + peer[n].host = ngx_pcalloc(cf->pool, + sizeof(ngx_http_upstream_host_t)); + if (peer[n].host == NULL) { + return NGX_ERROR; + } + + peer[n].host->name = server[i].host; + + peer[n].sockaddr = server[i].addrs[0].sockaddr; + peer[n].socklen = server[i].addrs[0].socklen; + peer[n].name = server[i].addrs[0].name; + peer[n].weight = server[i].weight; + peer[n].effective_weight = server[i].weight; + peer[n].current_weight = 0; + peer[n].max_conns = server[i].max_conns; + peer[n].max_fails = server[i].max_fails; + peer[n].fail_timeout = server[i].fail_timeout; + peer[n].down = server[i].down; + peer[n].server = server[i].name; + + *rpeerp = &peer[n]; + rpeerp = &peer[n].next; + n++; + + continue; + } +#endif + for (j = 0; j < server[i].naddrs; j++) { peer[n].sockaddr = server[i].addrs[j].sockaddr; peer[n].socklen = server[i].addrs[j].socklen; @@ -273,7 +426,12 @@ ngx_http_upstream_init_round_robin_peer(ngx_http_request_t *r, rrp->peers = us->peer.data; rrp->current = NULL; - rrp->config = 0; + + ngx_http_upstream_rr_peers_rlock(rrp->peers); + +#if (NGX_HTTP_UPSTREAM_ZONE) + rrp->config = rrp->peers->config ? *rrp->peers->config : 0; +#endif n = rrp->peers->number; @@ -281,6 +439,10 @@ ngx_http_upstream_init_round_robin_peer(ngx_http_request_t *r, n = rrp->peers->next->number; } + r->upstream->peer.tries = ngx_http_upstream_tries(rrp->peers); + + ngx_http_upstream_rr_peers_unlock(rrp->peers); + if (n <= 8 * sizeof(uintptr_t)) { rrp->tried = &rrp->data; rrp->data = 0; @@ -296,7 +458,6 @@ ngx_http_upstream_init_round_robin_peer(ngx_http_request_t *r, r->upstream->peer.get = ngx_http_upstream_get_round_robin_peer; r->upstream->peer.free = ngx_http_upstream_free_round_robin_peer; - r->upstream->peer.tries = ngx_http_upstream_tries(rrp->peers); #if (NGX_HTTP_SSL) r->upstream->peer.set_session = ngx_http_upstream_set_round_robin_peer_session; @@ -446,6 +607,12 @@ ngx_http_upstream_get_round_robin_peer(ngx_peer_connection_t *pc, void *data) peers = rrp->peers; ngx_http_upstream_rr_peers_wlock(peers); +#if (NGX_HTTP_UPSTREAM_ZONE) + if (peers->config && rrp->config != *peers->config) { + goto busy; + } +#endif + if (peers->single) { peer = peers->peer; @@ -458,6 +625,7 @@ ngx_http_upstream_get_round_robin_peer(ngx_peer_connection_t *pc, void *data) } rrp->current = peer; + ngx_http_upstream_rr_peer_ref(peers, peer); } else { @@ -510,6 +678,10 @@ ngx_http_upstream_get_round_robin_peer(ngx_peer_connection_t *pc, void *data) ngx_http_upstream_rr_peers_wlock(peers); } +#if (NGX_HTTP_UPSTREAM_ZONE) +busy: +#endif + ngx_http_upstream_rr_peers_unlock(peers); pc->name = peers->name; @@ -580,6 +752,7 @@ ngx_http_upstream_get_peer(ngx_http_upstream_rr_peer_data_t *rrp) } rrp->current = best; + ngx_http_upstream_rr_peer_ref(rrp->peers, best); n = p / (8 * sizeof(uintptr_t)); m = (uintptr_t) 1 << p % (8 * sizeof(uintptr_t)); @@ -617,9 +790,16 @@ ngx_http_upstream_free_round_robin_peer(ngx_peer_connection_t *pc, void *data, if (rrp->peers->single) { + if (peer->fails) { + peer->fails = 0; + } + peer->conns--; - ngx_http_upstream_rr_peer_unlock(rrp->peers, peer); + if (ngx_http_upstream_rr_peer_unref(rrp->peers, peer) == NGX_OK) { + ngx_http_upstream_rr_peer_unlock(rrp->peers, peer); + } + ngx_http_upstream_rr_peers_unlock(rrp->peers); pc->tries = 0; @@ -661,7 +841,10 @@ ngx_http_upstream_free_round_robin_peer(ngx_peer_connection_t *pc, void *data, peer->conns--; - ngx_http_upstream_rr_peer_unlock(rrp->peers, peer); + if (ngx_http_upstream_rr_peer_unref(rrp->peers, peer) == NGX_OK) { + ngx_http_upstream_rr_peer_unlock(rrp->peers, peer); + } + ngx_http_upstream_rr_peers_unlock(rrp->peers); if (pc->tries) { diff --git a/src/http/ngx_http_upstream_round_robin.h b/src/http/ngx_http_upstream_round_robin.h index 922ceaa04f2..06437f4059e 100644 --- a/src/http/ngx_http_upstream_round_robin.h +++ b/src/http/ngx_http_upstream_round_robin.h @@ -14,8 +14,23 @@ #include +typedef struct ngx_http_upstream_rr_peers_s ngx_http_upstream_rr_peers_t; typedef struct ngx_http_upstream_rr_peer_s ngx_http_upstream_rr_peer_t; + +#if (NGX_HTTP_UPSTREAM_ZONE) + +typedef struct { + ngx_event_t event; /* must be first */ + ngx_uint_t worker; + ngx_str_t name; + ngx_http_upstream_rr_peers_t *peers; + ngx_http_upstream_rr_peer_t *peer; +} ngx_http_upstream_host_t; + +#endif + + struct ngx_http_upstream_rr_peer_s { struct sockaddr *sockaddr; socklen_t socklen; @@ -46,24 +61,28 @@ struct ngx_http_upstream_rr_peer_s { #endif #if (NGX_HTTP_UPSTREAM_ZONE) + unsigned zombie:1; + ngx_atomic_t lock; + ngx_uint_t refs; + ngx_http_upstream_host_t *host; #endif ngx_http_upstream_rr_peer_t *next; - NGX_COMPAT_BEGIN(32) + NGX_COMPAT_BEGIN(15) NGX_COMPAT_END }; -typedef struct ngx_http_upstream_rr_peers_s ngx_http_upstream_rr_peers_t; - struct ngx_http_upstream_rr_peers_s { ngx_uint_t number; #if (NGX_HTTP_UPSTREAM_ZONE) ngx_slab_pool_t *shpool; ngx_atomic_t rwlock; + ngx_uint_t *config; + ngx_http_upstream_rr_peer_t *resolve; ngx_http_upstream_rr_peers_t *zone_next; #endif @@ -114,6 +133,65 @@ struct ngx_http_upstream_rr_peers_s { ngx_rwlock_unlock(&peer->lock); \ } + +#define ngx_http_upstream_rr_peer_ref(peers, peer) \ + (peer)->refs++; + + +static ngx_inline void +ngx_http_upstream_rr_peer_free_locked(ngx_http_upstream_rr_peers_t *peers, + ngx_http_upstream_rr_peer_t *peer) +{ + if (peer->refs) { + peer->zombie = 1; + return; + } + + ngx_slab_free_locked(peers->shpool, peer->sockaddr); + ngx_slab_free_locked(peers->shpool, peer->name.data); + + if (peer->server.data && (peer->host == NULL || peer->host->peer == peer)) { + ngx_slab_free_locked(peers->shpool, peer->server.data); + } + +#if (NGX_HTTP_SSL) + if (peer->ssl_session) { + ngx_slab_free_locked(peers->shpool, peer->ssl_session); + } +#endif + + ngx_slab_free_locked(peers->shpool, peer); +} + + +static ngx_inline void +ngx_http_upstream_rr_peer_free(ngx_http_upstream_rr_peers_t *peers, + ngx_http_upstream_rr_peer_t *peer) +{ + ngx_shmtx_lock(&peers->shpool->mutex); + ngx_http_upstream_rr_peer_free_locked(peers, peer); + ngx_shmtx_unlock(&peers->shpool->mutex); +} + + +static ngx_inline ngx_int_t +ngx_http_upstream_rr_peer_unref(ngx_http_upstream_rr_peers_t *peers, + ngx_http_upstream_rr_peer_t *peer) +{ + peer->refs--; + + if (peers->shpool == NULL) { + return NGX_OK; + } + + if (peer->refs == 0 && peer->zombie) { + ngx_http_upstream_rr_peer_free(peers, peer); + return NGX_DONE; + } + + return NGX_OK; +} + #else #define ngx_http_upstream_rr_peers_rlock(peers) @@ -121,6 +199,8 @@ struct ngx_http_upstream_rr_peers_s { #define ngx_http_upstream_rr_peers_unlock(peers) #define ngx_http_upstream_rr_peer_lock(peers, peer) #define ngx_http_upstream_rr_peer_unlock(peers, peer) +#define ngx_http_upstream_rr_peer_ref(peers, peer) +#define ngx_http_upstream_rr_peer_unref(peers, peer) NGX_OK #endif diff --git a/src/stream/ngx_stream_proxy_module.c b/src/stream/ngx_stream_proxy_module.c index bbf4f7ec002..0890a04aeb0 100644 --- a/src/stream/ngx_stream_proxy_module.c +++ b/src/stream/ngx_stream_proxy_module.c @@ -742,6 +742,25 @@ ngx_stream_proxy_connect(ngx_stream_session_t *s) u->state->peer = u->peer.name; +#if (NGX_STREAM_UPSTREAM_ZONE) + if (u->upstream && u->upstream->shm_zone + && (u->upstream->flags & NGX_STREAM_UPSTREAM_MODIFY)) + { + u->state->peer = ngx_palloc(s->connection->pool, + sizeof(ngx_str_t) + u->peer.name->len); + if (u->state->peer == NULL) { + ngx_stream_proxy_finalize(s, NGX_STREAM_INTERNAL_SERVER_ERROR); + return; + } + + u->state->peer->len = u->peer.name->len; + u->state->peer->data = (u_char *) (u->state->peer + 1); + ngx_memcpy(u->state->peer->data, u->peer.name->data, u->peer.name->len); + + u->peer.name = u->state->peer; + } +#endif + if (rc == NGX_BUSY) { ngx_log_error(NGX_LOG_ERR, c->log, 0, "no live upstreams"); ngx_stream_proxy_finalize(s, NGX_STREAM_BAD_GATEWAY); diff --git a/src/stream/ngx_stream_upstream.c b/src/stream/ngx_stream_upstream.c index eadcf9f9a0d..a251cca0062 100644 --- a/src/stream/ngx_stream_upstream.c +++ b/src/stream/ngx_stream_upstream.c @@ -319,6 +319,7 @@ ngx_stream_upstream(ngx_conf_t *cf, ngx_command_t *cmd, void *dummy) u.no_port = 1; uscf = ngx_stream_upstream_add(cf, &u, NGX_STREAM_UPSTREAM_CREATE + |NGX_STREAM_UPSTREAM_MODIFY |NGX_STREAM_UPSTREAM_WEIGHT |NGX_STREAM_UPSTREAM_MAX_CONNS |NGX_STREAM_UPSTREAM_MAX_FAILS @@ -408,6 +409,9 @@ ngx_stream_upstream_server(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) ngx_url_t u; ngx_int_t weight, max_conns, max_fails; ngx_uint_t i; +#if (NGX_STREAM_UPSTREAM_ZONE) + ngx_uint_t resolve; +#endif ngx_stream_upstream_server_t *us; us = ngx_array_push(uscf->servers); @@ -423,6 +427,9 @@ ngx_stream_upstream_server(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) max_conns = 0; max_fails = 1; fail_timeout = 10; +#if (NGX_STREAM_UPSTREAM_ZONE) + resolve = 0; +#endif for (i = 2; i < cf->args->nelts; i++) { @@ -511,6 +518,13 @@ ngx_stream_upstream_server(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) continue; } +#if (NGX_STREAM_UPSTREAM_ZONE) + if (ngx_strcmp(value[i].data, "resolve") == 0) { + resolve = 1; + continue; + } +#endif + goto invalid; } @@ -518,6 +532,13 @@ ngx_stream_upstream_server(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) u.url = value[1]; +#if (NGX_STREAM_UPSTREAM_ZONE) + if (resolve) { + /* resolve at run time */ + u.no_resolve = 1; + } +#endif + if (ngx_parse_url(cf->pool, &u) != NGX_OK) { if (u.err) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, @@ -534,8 +555,45 @@ ngx_stream_upstream_server(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) } us->name = u.url; + +#if (NGX_STREAM_UPSTREAM_ZONE) + + if (resolve && u.naddrs == 0) { + ngx_addr_t *addr; + + /* save port */ + + addr = ngx_pcalloc(cf->pool, sizeof(ngx_addr_t)); + if (addr == NULL) { + return NGX_CONF_ERROR; + } + + addr->sockaddr = ngx_palloc(cf->pool, u.socklen); + if (addr->sockaddr == NULL) { + return NGX_CONF_ERROR; + } + + ngx_memcpy(addr->sockaddr, &u.sockaddr, u.socklen); + + addr->socklen = u.socklen; + + us->addrs = addr; + us->naddrs = 1; + + us->host = u.host; + + } else { + us->addrs = u.addrs; + us->naddrs = u.naddrs; + } + +#else + us->addrs = u.addrs; us->naddrs = u.naddrs; + +#endif + us->weight = weight; us->max_conns = max_conns; us->max_fails = max_fails; diff --git a/src/stream/ngx_stream_upstream.h b/src/stream/ngx_stream_upstream.h index f5617794fed..686e9269c44 100644 --- a/src/stream/ngx_stream_upstream.h +++ b/src/stream/ngx_stream_upstream.h @@ -21,6 +21,7 @@ #define NGX_STREAM_UPSTREAM_FAIL_TIMEOUT 0x0008 #define NGX_STREAM_UPSTREAM_DOWN 0x0010 #define NGX_STREAM_UPSTREAM_BACKUP 0x0020 +#define NGX_STREAM_UPSTREAM_MODIFY 0x0040 #define NGX_STREAM_UPSTREAM_MAX_CONNS 0x0100 @@ -62,7 +63,11 @@ typedef struct { unsigned backup:1; - NGX_COMPAT_BEGIN(4) +#if (NGX_STREAM_UPSTREAM_ZONE) + ngx_str_t host; +#endif + + NGX_COMPAT_BEGIN(2) NGX_COMPAT_END } ngx_stream_upstream_server_t; @@ -83,6 +88,8 @@ struct ngx_stream_upstream_srv_conf_s { #if (NGX_STREAM_UPSTREAM_ZONE) ngx_shm_zone_t *shm_zone; + ngx_resolver_t *resolver; + ngx_msec_t resolver_timeout; #endif }; diff --git a/src/stream/ngx_stream_upstream_hash_module.c b/src/stream/ngx_stream_upstream_hash_module.c index b764fcbe0d2..20fca6c0d88 100644 --- a/src/stream/ngx_stream_upstream_hash_module.c +++ b/src/stream/ngx_stream_upstream_hash_module.c @@ -23,6 +23,9 @@ typedef struct { typedef struct { +#if (NGX_STREAM_UPSTREAM_ZONE) + ngx_uint_t config; +#endif ngx_stream_complex_value_t key; ngx_stream_upstream_chash_points_t *points; } ngx_stream_upstream_hash_srv_conf_t; @@ -49,6 +52,8 @@ static ngx_int_t ngx_stream_upstream_get_hash_peer(ngx_peer_connection_t *pc, static ngx_int_t ngx_stream_upstream_init_chash(ngx_conf_t *cf, ngx_stream_upstream_srv_conf_t *us); +static ngx_int_t ngx_stream_upstream_update_chash(ngx_pool_t *pool, + ngx_stream_upstream_srv_conf_t *us); static int ngx_libc_cdecl ngx_stream_upstream_chash_cmp_points(const void *one, const void *two); static ngx_uint_t ngx_stream_upstream_find_chash_point( @@ -178,10 +183,17 @@ ngx_stream_upstream_get_hash_peer(ngx_peer_connection_t *pc, void *data) ngx_stream_upstream_rr_peers_rlock(hp->rrp.peers); - if (hp->tries > 20 || hp->rrp.peers->single || hp->key.len == 0) { + if (hp->tries > 20 || hp->rrp.peers->number < 2 || hp->key.len == 0) { + ngx_stream_upstream_rr_peers_unlock(hp->rrp.peers); + return hp->get_rr_peer(pc, &hp->rrp); + } + +#if (NGX_STREAM_UPSTREAM_ZONE) + if (hp->rrp.peers->config && hp->rrp.config != *hp->rrp.peers->config) { ngx_stream_upstream_rr_peers_unlock(hp->rrp.peers); return hp->get_rr_peer(pc, &hp->rrp); } +#endif now = ngx_time(); @@ -261,6 +273,7 @@ ngx_stream_upstream_get_hash_peer(ngx_peer_connection_t *pc, void *data) } hp->rrp.current = peer; + ngx_stream_upstream_rr_peer_ref(hp->rrp.peers, peer); pc->sockaddr = peer->sockaddr; pc->socklen = peer->socklen; @@ -284,6 +297,26 @@ ngx_stream_upstream_get_hash_peer(ngx_peer_connection_t *pc, void *data) static ngx_int_t ngx_stream_upstream_init_chash(ngx_conf_t *cf, ngx_stream_upstream_srv_conf_t *us) +{ + if (ngx_stream_upstream_init_round_robin(cf, us) != NGX_OK) { + return NGX_ERROR; + } + + us->peer.init = ngx_stream_upstream_init_chash_peer; + +#if (NGX_STREAM_UPSTREAM_ZONE) + if (us->shm_zone) { + return NGX_OK; + } +#endif + + return ngx_stream_upstream_update_chash(cf->pool, us); +} + + +static ngx_int_t +ngx_stream_upstream_update_chash(ngx_pool_t *pool, + ngx_stream_upstream_srv_conf_t *us) { u_char *host, *port, c; size_t host_len, port_len, size; @@ -299,25 +332,33 @@ ngx_stream_upstream_init_chash(ngx_conf_t *cf, u_char byte[4]; } prev_hash; - if (ngx_stream_upstream_init_round_robin(cf, us) != NGX_OK) { - return NGX_ERROR; - } + hcf = ngx_stream_conf_upstream_srv_conf(us, + ngx_stream_upstream_hash_module); - us->peer.init = ngx_stream_upstream_init_chash_peer; + if (hcf->points) { + ngx_free(hcf->points); + hcf->points = NULL; + } peers = us->peer.data; npoints = peers->total_weight * 160; size = sizeof(ngx_stream_upstream_chash_points_t) - + sizeof(ngx_stream_upstream_chash_point_t) * (npoints - 1); + - sizeof(ngx_stream_upstream_chash_point_t) + + sizeof(ngx_stream_upstream_chash_point_t) * npoints; - points = ngx_palloc(cf->pool, size); + points = pool ? ngx_palloc(pool, size) : ngx_alloc(size, ngx_cycle->log); if (points == NULL) { return NGX_ERROR; } points->number = 0; + if (npoints == 0) { + hcf->points = points; + return NGX_OK; + } + for (peer = peers->peer; peer; peer = peer->next) { server = &peer->server; @@ -401,8 +442,6 @@ ngx_stream_upstream_init_chash(ngx_conf_t *cf, points->number = i + 1; - hcf = ngx_stream_conf_upstream_srv_conf(us, - ngx_stream_upstream_hash_module); hcf->points = points; return NGX_OK; @@ -483,7 +522,22 @@ ngx_stream_upstream_init_chash_peer(ngx_stream_session_t *s, ngx_stream_upstream_rr_peers_rlock(hp->rrp.peers); - hp->hash = ngx_stream_upstream_find_chash_point(hcf->points, hash); +#if (NGX_STREAM_UPSTREAM_ZONE) + if (hp->rrp.peers->config + && (hcf->points == NULL || hcf->config != *hp->rrp.peers->config)) + { + if (ngx_stream_upstream_update_chash(NULL, us) != NGX_OK) { + ngx_stream_upstream_rr_peers_unlock(hp->rrp.peers); + return NGX_ERROR; + } + + hcf->config = *hp->rrp.peers->config; + } +#endif + + if (hcf->points->number) { + hp->hash = ngx_stream_upstream_find_chash_point(hcf->points, hash); + } ngx_stream_upstream_rr_peers_unlock(hp->rrp.peers); @@ -518,6 +572,20 @@ ngx_stream_upstream_get_chash_peer(ngx_peer_connection_t *pc, void *data) pc->connection = NULL; + if (hp->rrp.peers->number == 0) { + pc->name = hp->rrp.peers->name; + ngx_stream_upstream_rr_peers_unlock(hp->rrp.peers); + return NGX_BUSY; + } + +#if (NGX_STREAM_UPSTREAM_ZONE) + if (hp->rrp.peers->config && hp->rrp.config != *hp->rrp.peers->config) { + pc->name = hp->rrp.peers->name; + ngx_stream_upstream_rr_peers_unlock(hp->rrp.peers); + return NGX_BUSY; + } +#endif + now = ngx_time(); hcf = hp->conf; @@ -596,6 +664,7 @@ ngx_stream_upstream_get_chash_peer(ngx_peer_connection_t *pc, void *data) } hp->rrp.current = best; + ngx_stream_upstream_rr_peer_ref(hp->rrp.peers, best); pc->sockaddr = best->sockaddr; pc->socklen = best->socklen; @@ -663,6 +732,7 @@ ngx_stream_upstream_hash(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) } uscf->flags = NGX_STREAM_UPSTREAM_CREATE + |NGX_STREAM_UPSTREAM_MODIFY |NGX_STREAM_UPSTREAM_WEIGHT |NGX_STREAM_UPSTREAM_MAX_CONNS |NGX_STREAM_UPSTREAM_MAX_FAILS diff --git a/src/stream/ngx_stream_upstream_least_conn_module.c b/src/stream/ngx_stream_upstream_least_conn_module.c index 739b20a9f1e..6e8360440e1 100644 --- a/src/stream/ngx_stream_upstream_least_conn_module.c +++ b/src/stream/ngx_stream_upstream_least_conn_module.c @@ -120,6 +120,12 @@ ngx_stream_upstream_get_least_conn_peer(ngx_peer_connection_t *pc, void *data) ngx_stream_upstream_rr_peers_wlock(peers); +#if (NGX_STREAM_UPSTREAM_ZONE) + if (peers->config && rrp->config != *peers->config) { + goto busy; + } +#endif + best = NULL; total = 0; @@ -240,6 +246,7 @@ ngx_stream_upstream_get_least_conn_peer(ngx_peer_connection_t *pc, void *data) best->conns++; rrp->current = best; + ngx_stream_upstream_rr_peer_ref(peers, best); n = p / (8 * sizeof(uintptr_t)); m = (uintptr_t) 1 << p % (8 * sizeof(uintptr_t)); @@ -276,6 +283,10 @@ ngx_stream_upstream_get_least_conn_peer(ngx_peer_connection_t *pc, void *data) ngx_stream_upstream_rr_peers_wlock(peers); } +#if (NGX_STREAM_UPSTREAM_ZONE) +busy: +#endif + ngx_stream_upstream_rr_peers_unlock(peers); pc->name = peers->name; @@ -299,6 +310,7 @@ ngx_stream_upstream_least_conn(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) uscf->peer.init_upstream = ngx_stream_upstream_init_least_conn; uscf->flags = NGX_STREAM_UPSTREAM_CREATE + |NGX_STREAM_UPSTREAM_MODIFY |NGX_STREAM_UPSTREAM_WEIGHT |NGX_STREAM_UPSTREAM_MAX_CONNS |NGX_STREAM_UPSTREAM_MAX_FAILS diff --git a/src/stream/ngx_stream_upstream_random_module.c b/src/stream/ngx_stream_upstream_random_module.c index 402c1b29890..71f76f8d822 100644 --- a/src/stream/ngx_stream_upstream_random_module.c +++ b/src/stream/ngx_stream_upstream_random_module.c @@ -17,6 +17,9 @@ typedef struct { typedef struct { ngx_uint_t two; +#if (NGX_STREAM_UPSTREAM_ZONE) + ngx_uint_t config; +#endif ngx_stream_upstream_random_range_t *ranges; } ngx_stream_upstream_random_srv_conf_t; @@ -125,6 +128,11 @@ ngx_stream_upstream_update_random(ngx_pool_t *pool, rcf = ngx_stream_conf_upstream_srv_conf(us, ngx_stream_upstream_random_module); + if (rcf->ranges) { + ngx_free(rcf->ranges); + rcf->ranges = NULL; + } + peers = us->peer.data; size = peers->number * sizeof(ngx_stream_upstream_random_range_t); @@ -186,11 +194,15 @@ ngx_stream_upstream_init_random_peer(ngx_stream_session_t *s, ngx_stream_upstream_rr_peers_rlock(rp->rrp.peers); #if (NGX_STREAM_UPSTREAM_ZONE) - if (rp->rrp.peers->shpool && rcf->ranges == NULL) { + if (rp->rrp.peers->config + && (rcf->ranges == NULL || rcf->config != *rp->rrp.peers->config)) + { if (ngx_stream_upstream_update_random(NULL, us) != NGX_OK) { ngx_stream_upstream_rr_peers_unlock(rp->rrp.peers); return NGX_ERROR; } + + rcf->config = *rp->rrp.peers->config; } #endif @@ -220,11 +232,18 @@ ngx_stream_upstream_get_random_peer(ngx_peer_connection_t *pc, void *data) ngx_stream_upstream_rr_peers_rlock(peers); - if (rp->tries > 20 || peers->single) { + if (rp->tries > 20 || peers->number < 2) { ngx_stream_upstream_rr_peers_unlock(peers); return ngx_stream_upstream_get_round_robin_peer(pc, rrp); } +#if (NGX_STREAM_UPSTREAM_ZONE) + if (peers->config && rrp->config != *peers->config) { + ngx_stream_upstream_rr_peers_unlock(peers); + return ngx_stream_upstream_get_round_robin_peer(pc, rrp); + } +#endif + pc->cached = 0; pc->connection = NULL; @@ -274,6 +293,7 @@ ngx_stream_upstream_get_random_peer(ngx_peer_connection_t *pc, void *data) } rrp->current = peer; + ngx_stream_upstream_rr_peer_ref(peers, peer); if (now - peer->checked > peer->fail_timeout) { peer->checked = now; @@ -314,10 +334,17 @@ ngx_stream_upstream_get_random2_peer(ngx_peer_connection_t *pc, void *data) ngx_stream_upstream_rr_peers_wlock(peers); - if (rp->tries > 20 || peers->single) { + if (rp->tries > 20 || peers->number < 2) { + ngx_stream_upstream_rr_peers_unlock(peers); + return ngx_stream_upstream_get_round_robin_peer(pc, rrp); + } + +#if (NGX_STREAM_UPSTREAM_ZONE) + if (peers->config && rrp->config != *peers->config) { ngx_stream_upstream_rr_peers_unlock(peers); return ngx_stream_upstream_get_round_robin_peer(pc, rrp); } +#endif pc->cached = 0; pc->connection = NULL; @@ -384,6 +411,7 @@ ngx_stream_upstream_get_random2_peer(ngx_peer_connection_t *pc, void *data) } rrp->current = peer; + ngx_stream_upstream_rr_peer_ref(peers, peer); if (now - peer->checked > peer->fail_timeout) { peer->checked = now; @@ -467,6 +495,7 @@ ngx_stream_upstream_random(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) uscf->peer.init_upstream = ngx_stream_upstream_init_random; uscf->flags = NGX_STREAM_UPSTREAM_CREATE + |NGX_STREAM_UPSTREAM_MODIFY |NGX_STREAM_UPSTREAM_WEIGHT |NGX_STREAM_UPSTREAM_MAX_CONNS |NGX_STREAM_UPSTREAM_MAX_FAILS diff --git a/src/stream/ngx_stream_upstream_round_robin.c b/src/stream/ngx_stream_upstream_round_robin.c index ae3bf37a68a..a18171a3957 100644 --- a/src/stream/ngx_stream_upstream_round_robin.c +++ b/src/stream/ngx_stream_upstream_round_robin.c @@ -38,10 +38,15 @@ ngx_stream_upstream_init_round_robin(ngx_conf_t *cf, ngx_stream_upstream_srv_conf_t *us) { ngx_url_t u; - ngx_uint_t i, j, n, w, t; + ngx_uint_t i, j, n, r, w, t; ngx_stream_upstream_server_t *server; ngx_stream_upstream_rr_peer_t *peer, **peerp; ngx_stream_upstream_rr_peers_t *peers, *backup; +#if (NGX_STREAM_UPSTREAM_ZONE) + ngx_uint_t resolve; + ngx_stream_core_srv_conf_t *cscf; + ngx_stream_upstream_rr_peer_t **rpeerp; +#endif us->peer.init = ngx_stream_upstream_init_round_robin_peer; @@ -49,14 +54,33 @@ ngx_stream_upstream_init_round_robin(ngx_conf_t *cf, server = us->servers->elts; n = 0; + r = 0; w = 0; t = 0; +#if (NGX_STREAM_UPSTREAM_ZONE) + resolve = 0; +#endif + for (i = 0; i < us->servers->nelts; i++) { + +#if (NGX_STREAM_UPSTREAM_ZONE) + if (server[i].host.len) { + resolve = 1; + } +#endif + if (server[i].backup) { continue; } +#if (NGX_STREAM_UPSTREAM_ZONE) + if (server[i].host.len) { + r++; + continue; + } +#endif + n += server[i].naddrs; w += server[i].naddrs * server[i].weight; @@ -65,7 +89,54 @@ ngx_stream_upstream_init_round_robin(ngx_conf_t *cf, } } - if (n == 0) { +#if (NGX_STREAM_UPSTREAM_ZONE) + if (us->shm_zone) { + + if (resolve && !(us->flags & NGX_STREAM_UPSTREAM_MODIFY)) { + ngx_log_error(NGX_LOG_EMERG, cf->log, 0, + "load balancing method does not support" + " resolving names at run time in" + " upstream \"%V\" in %s:%ui", + &us->host, us->file_name, us->line); + return NGX_ERROR; + } + + cscf = ngx_stream_conf_get_module_srv_conf(cf, + ngx_stream_core_module); + + us->resolver = cscf->resolver; + us->resolver_timeout = cscf->resolver_timeout; + + /* + * Without "resolver_timeout" in stream{}, the value is unset. + * Even if we set it in ngx_stream_core_merge_srv_conf(), it's + * still dependent on the module order and unreliable. + */ + ngx_conf_init_msec_value(us->resolver_timeout, 30000); + + if (resolve + && (us->resolver == NULL + || us->resolver->connections.nelts == 0)) + { + ngx_log_error(NGX_LOG_EMERG, cf->log, 0, + "no resolver defined to resolve names" + " at run time in upstream \"%V\" in %s:%ui", + &us->host, us->file_name, us->line); + return NGX_ERROR; + } + + } else if (resolve) { + + ngx_log_error(NGX_LOG_EMERG, cf->log, 0, + "resolving names at run time requires" + " upstream \"%V\" in %s:%ui" + " to be in shared memory", + &us->host, us->file_name, us->line); + return NGX_ERROR; + } +#endif + + if (n + r == 0) { ngx_log_error(NGX_LOG_EMERG, cf->log, 0, "no servers in upstream \"%V\" in %s:%ui", &us->host, us->file_name, us->line); @@ -77,7 +148,8 @@ ngx_stream_upstream_init_round_robin(ngx_conf_t *cf, return NGX_ERROR; } - peer = ngx_pcalloc(cf->pool, sizeof(ngx_stream_upstream_rr_peer_t) * n); + peer = ngx_pcalloc(cf->pool, sizeof(ngx_stream_upstream_rr_peer_t) + * (n + r)); if (peer == NULL) { return NGX_ERROR; } @@ -92,11 +164,45 @@ ngx_stream_upstream_init_round_robin(ngx_conf_t *cf, n = 0; peerp = &peers->peer; +#if (NGX_STREAM_UPSTREAM_ZONE) + rpeerp = &peers->resolve; +#endif + for (i = 0; i < us->servers->nelts; i++) { if (server[i].backup) { continue; } +#if (NGX_STREAM_UPSTREAM_ZONE) + if (server[i].host.len) { + + peer[n].host = ngx_pcalloc(cf->pool, + sizeof(ngx_stream_upstream_host_t)); + if (peer[n].host == NULL) { + return NGX_ERROR; + } + + peer[n].host->name = server[i].host; + + peer[n].sockaddr = server[i].addrs[0].sockaddr; + peer[n].socklen = server[i].addrs[0].socklen; + peer[n].name = server[i].addrs[0].name; + peer[n].weight = server[i].weight; + peer[n].effective_weight = server[i].weight; + peer[n].current_weight = 0; + peer[n].max_conns = server[i].max_conns; + peer[n].max_fails = server[i].max_fails; + peer[n].fail_timeout = server[i].fail_timeout; + peer[n].down = server[i].down; + peer[n].server = server[i].name; + *rpeerp = &peer[n]; + rpeerp = &peer[n].next; + n++; + + continue; + } +#endif + for (j = 0; j < server[i].naddrs; j++) { peer[n].sockaddr = server[i].addrs[j].sockaddr; peer[n].socklen = server[i].addrs[j].socklen; @@ -121,6 +227,7 @@ ngx_stream_upstream_init_round_robin(ngx_conf_t *cf, /* backup servers */ n = 0; + r = 0; w = 0; t = 0; @@ -129,6 +236,13 @@ ngx_stream_upstream_init_round_robin(ngx_conf_t *cf, continue; } +#if (NGX_STREAM_UPSTREAM_ZONE) + if (server[i].host.len) { + r++; + continue; + } +#endif + n += server[i].naddrs; w += server[i].naddrs * server[i].weight; @@ -137,7 +251,7 @@ ngx_stream_upstream_init_round_robin(ngx_conf_t *cf, } } - if (n == 0) { + if (n + r == 0) { return NGX_OK; } @@ -146,12 +260,16 @@ ngx_stream_upstream_init_round_robin(ngx_conf_t *cf, return NGX_ERROR; } - peer = ngx_pcalloc(cf->pool, sizeof(ngx_stream_upstream_rr_peer_t) * n); + peer = ngx_pcalloc(cf->pool, sizeof(ngx_stream_upstream_rr_peer_t) + * (n + r)); if (peer == NULL) { return NGX_ERROR; } - peers->single = 0; + if (n > 0) { + peers->single = 0; + } + backup->single = 0; backup->number = n; backup->weighted = (w != n); @@ -162,11 +280,45 @@ ngx_stream_upstream_init_round_robin(ngx_conf_t *cf, n = 0; peerp = &backup->peer; +#if (NGX_STREAM_UPSTREAM_ZONE) + rpeerp = &backup->resolve; +#endif + for (i = 0; i < us->servers->nelts; i++) { if (!server[i].backup) { continue; } +#if (NGX_STREAM_UPSTREAM_ZONE) + if (server[i].host.len) { + + peer[n].host = ngx_pcalloc(cf->pool, + sizeof(ngx_stream_upstream_host_t)); + if (peer[n].host == NULL) { + return NGX_ERROR; + } + + peer[n].host->name = server[i].host; + + peer[n].sockaddr = server[i].addrs[0].sockaddr; + peer[n].socklen = server[i].addrs[0].socklen; + peer[n].name = server[i].addrs[0].name; + peer[n].weight = server[i].weight; + peer[n].effective_weight = server[i].weight; + peer[n].current_weight = 0; + peer[n].max_conns = server[i].max_conns; + peer[n].max_fails = server[i].max_fails; + peer[n].fail_timeout = server[i].fail_timeout; + peer[n].down = server[i].down; + peer[n].server = server[i].name; + *rpeerp = &peer[n]; + rpeerp = &peer[n].next; + n++; + + continue; + } +#endif + for (j = 0; j < server[i].naddrs; j++) { peer[n].sockaddr = server[i].addrs[j].sockaddr; peer[n].socklen = server[i].addrs[j].socklen; @@ -280,7 +432,12 @@ ngx_stream_upstream_init_round_robin_peer(ngx_stream_session_t *s, rrp->peers = us->peer.data; rrp->current = NULL; - rrp->config = 0; + + ngx_stream_upstream_rr_peers_rlock(rrp->peers); + +#if (NGX_STREAM_UPSTREAM_ZONE) + rrp->config = rrp->peers->config ? *rrp->peers->config : 0; +#endif n = rrp->peers->number; @@ -288,6 +445,10 @@ ngx_stream_upstream_init_round_robin_peer(ngx_stream_session_t *s, n = rrp->peers->next->number; } + s->upstream->peer.tries = ngx_stream_upstream_tries(rrp->peers); + + ngx_stream_upstream_rr_peers_unlock(rrp->peers); + if (n <= 8 * sizeof(uintptr_t)) { rrp->tried = &rrp->data; rrp->data = 0; @@ -304,7 +465,7 @@ ngx_stream_upstream_init_round_robin_peer(ngx_stream_session_t *s, s->upstream->peer.get = ngx_stream_upstream_get_round_robin_peer; s->upstream->peer.free = ngx_stream_upstream_free_round_robin_peer; s->upstream->peer.notify = ngx_stream_upstream_notify_round_robin_peer; - s->upstream->peer.tries = ngx_stream_upstream_tries(rrp->peers); + #if (NGX_STREAM_SSL) s->upstream->peer.set_session = ngx_stream_upstream_set_round_robin_peer_session; @@ -455,6 +616,12 @@ ngx_stream_upstream_get_round_robin_peer(ngx_peer_connection_t *pc, void *data) peers = rrp->peers; ngx_stream_upstream_rr_peers_wlock(peers); +#if (NGX_STREAM_UPSTREAM_ZONE) + if (peers->config && rrp->config != *peers->config) { + goto busy; + } +#endif + if (peers->single) { peer = peers->peer; @@ -467,6 +634,7 @@ ngx_stream_upstream_get_round_robin_peer(ngx_peer_connection_t *pc, void *data) } rrp->current = peer; + ngx_stream_upstream_rr_peer_ref(peers, peer); } else { @@ -519,6 +687,10 @@ ngx_stream_upstream_get_round_robin_peer(ngx_peer_connection_t *pc, void *data) ngx_stream_upstream_rr_peers_wlock(peers); } +#if (NGX_STREAM_UPSTREAM_ZONE) +busy: +#endif + ngx_stream_upstream_rr_peers_unlock(peers); pc->name = peers->name; @@ -589,6 +761,7 @@ ngx_stream_upstream_get_peer(ngx_stream_upstream_rr_peer_data_t *rrp) } rrp->current = best; + ngx_stream_upstream_rr_peer_ref(rrp->peers, best); n = p / (8 * sizeof(uintptr_t)); m = (uintptr_t) 1 << p % (8 * sizeof(uintptr_t)); @@ -623,9 +796,17 @@ ngx_stream_upstream_free_round_robin_peer(ngx_peer_connection_t *pc, void *data, ngx_stream_upstream_rr_peer_lock(rrp->peers, peer); if (rrp->peers->single) { + + if (peer->fails) { + peer->fails = 0; + } + peer->conns--; - ngx_stream_upstream_rr_peer_unlock(rrp->peers, peer); + if (ngx_stream_upstream_rr_peer_unref(rrp->peers, peer) == NGX_OK) { + ngx_stream_upstream_rr_peer_unlock(rrp->peers, peer); + } + ngx_stream_upstream_rr_peers_unlock(rrp->peers); pc->tries = 0; @@ -667,7 +848,10 @@ ngx_stream_upstream_free_round_robin_peer(ngx_peer_connection_t *pc, void *data, peer->conns--; - ngx_stream_upstream_rr_peer_unlock(rrp->peers, peer); + if (ngx_stream_upstream_rr_peer_unref(rrp->peers, peer) == NGX_OK) { + ngx_stream_upstream_rr_peer_unlock(rrp->peers, peer); + } + ngx_stream_upstream_rr_peers_unlock(rrp->peers); if (pc->tries) { diff --git a/src/stream/ngx_stream_upstream_round_robin.h b/src/stream/ngx_stream_upstream_round_robin.h index bd96667ba36..5002200c26a 100644 --- a/src/stream/ngx_stream_upstream_round_robin.h +++ b/src/stream/ngx_stream_upstream_round_robin.h @@ -14,8 +14,23 @@ #include +typedef struct ngx_stream_upstream_rr_peers_s ngx_stream_upstream_rr_peers_t; typedef struct ngx_stream_upstream_rr_peer_s ngx_stream_upstream_rr_peer_t; + +#if (NGX_STREAM_UPSTREAM_ZONE) + +typedef struct { + ngx_event_t event; /* must be first */ + ngx_uint_t worker; + ngx_str_t name; + ngx_stream_upstream_rr_peers_t *peers; + ngx_stream_upstream_rr_peer_t *peer; +} ngx_stream_upstream_host_t; + +#endif + + struct ngx_stream_upstream_rr_peer_s { struct sockaddr *sockaddr; socklen_t socklen; @@ -44,24 +59,28 @@ struct ngx_stream_upstream_rr_peer_s { int ssl_session_len; #if (NGX_STREAM_UPSTREAM_ZONE) + unsigned zombie:1; + ngx_atomic_t lock; + ngx_uint_t refs; + ngx_stream_upstream_host_t *host; #endif ngx_stream_upstream_rr_peer_t *next; - NGX_COMPAT_BEGIN(25) + NGX_COMPAT_BEGIN(14) NGX_COMPAT_END }; -typedef struct ngx_stream_upstream_rr_peers_s ngx_stream_upstream_rr_peers_t; - struct ngx_stream_upstream_rr_peers_s { ngx_uint_t number; #if (NGX_STREAM_UPSTREAM_ZONE) ngx_slab_pool_t *shpool; ngx_atomic_t rwlock; + ngx_uint_t *config; + ngx_stream_upstream_rr_peer_t *resolve; ngx_stream_upstream_rr_peers_t *zone_next; #endif @@ -112,6 +131,65 @@ struct ngx_stream_upstream_rr_peers_s { ngx_rwlock_unlock(&peer->lock); \ } + +#define ngx_stream_upstream_rr_peer_ref(peers, peer) \ + (peer)->refs++; + + +static ngx_inline void +ngx_stream_upstream_rr_peer_free_locked(ngx_stream_upstream_rr_peers_t *peers, + ngx_stream_upstream_rr_peer_t *peer) +{ + if (peer->refs) { + peer->zombie = 1; + return; + } + + ngx_slab_free_locked(peers->shpool, peer->sockaddr); + ngx_slab_free_locked(peers->shpool, peer->name.data); + + if (peer->server.data && (peer->host == NULL || peer->host->peer == peer)) { + ngx_slab_free_locked(peers->shpool, peer->server.data); + } + +#if (NGX_STREAM_SSL) + if (peer->ssl_session) { + ngx_slab_free_locked(peers->shpool, peer->ssl_session); + } +#endif + + ngx_slab_free_locked(peers->shpool, peer); +} + + +static ngx_inline void +ngx_stream_upstream_rr_peer_free(ngx_stream_upstream_rr_peers_t *peers, + ngx_stream_upstream_rr_peer_t *peer) +{ + ngx_shmtx_lock(&peers->shpool->mutex); + ngx_stream_upstream_rr_peer_free_locked(peers, peer); + ngx_shmtx_unlock(&peers->shpool->mutex); +} + + +static ngx_inline ngx_int_t +ngx_stream_upstream_rr_peer_unref(ngx_stream_upstream_rr_peers_t *peers, + ngx_stream_upstream_rr_peer_t *peer) +{ + peer->refs--; + + if (peers->shpool == NULL) { + return NGX_OK; + } + + if (peer->refs == 0 && peer->zombie) { + ngx_stream_upstream_rr_peer_free(peers, peer); + return NGX_DONE; + } + + return NGX_OK; +} + #else #define ngx_stream_upstream_rr_peers_rlock(peers) @@ -119,6 +197,8 @@ struct ngx_stream_upstream_rr_peers_s { #define ngx_stream_upstream_rr_peers_unlock(peers) #define ngx_stream_upstream_rr_peer_lock(peers, peer) #define ngx_stream_upstream_rr_peer_unlock(peers, peer) +#define ngx_stream_upstream_rr_peer_ref(peers, peer) +#define ngx_stream_upstream_rr_peer_unref(peers, peer) NGX_OK #endif diff --git a/src/stream/ngx_stream_upstream_zone_module.c b/src/stream/ngx_stream_upstream_zone_module.c index 80d42fa0018..d731baf317c 100644 --- a/src/stream/ngx_stream_upstream_zone_module.c +++ b/src/stream/ngx_stream_upstream_zone_module.c @@ -18,6 +18,13 @@ static ngx_stream_upstream_rr_peers_t *ngx_stream_upstream_zone_copy_peers( ngx_slab_pool_t *shpool, ngx_stream_upstream_srv_conf_t *uscf); static ngx_stream_upstream_rr_peer_t *ngx_stream_upstream_zone_copy_peer( ngx_stream_upstream_rr_peers_t *peers, ngx_stream_upstream_rr_peer_t *src); +static void ngx_stream_upstream_zone_set_single( + ngx_stream_upstream_srv_conf_t *uscf); +static void ngx_stream_upstream_zone_remove_peer_locked( + ngx_stream_upstream_rr_peers_t *peers, ngx_stream_upstream_rr_peer_t *peer); +static ngx_int_t ngx_stream_upstream_zone_init_worker(ngx_cycle_t *cycle); +static void ngx_stream_upstream_zone_resolve_timer(ngx_event_t *event); +static void ngx_stream_upstream_zone_resolve_handler(ngx_resolver_ctx_t *ctx); static ngx_command_t ngx_stream_upstream_zone_commands[] = { @@ -52,7 +59,7 @@ ngx_module_t ngx_stream_upstream_zone_module = { NGX_STREAM_MODULE, /* module type */ NULL, /* init master */ NULL, /* init module */ - NULL, /* init process */ + ngx_stream_upstream_zone_init_worker, /* init process */ NULL, /* init thread */ NULL, /* exit thread */ NULL, /* exit process */ @@ -185,9 +192,15 @@ ngx_stream_upstream_zone_copy_peers(ngx_slab_pool_t *shpool, ngx_stream_upstream_srv_conf_t *uscf) { ngx_str_t *name; + ngx_uint_t *config; ngx_stream_upstream_rr_peer_t *peer, **peerp; ngx_stream_upstream_rr_peers_t *peers, *backup; + config = ngx_slab_calloc(shpool, sizeof(ngx_uint_t)); + if (config == NULL) { + return NULL; + } + peers = ngx_slab_alloc(shpool, sizeof(ngx_stream_upstream_rr_peers_t)); if (peers == NULL) { return NULL; @@ -211,6 +224,7 @@ ngx_stream_upstream_zone_copy_peers(ngx_slab_pool_t *shpool, peers->name = name; peers->shpool = shpool; + peers->config = config; for (peerp = &peers->peer; *peerp; peerp = &peer->next) { /* pool is unlocked */ @@ -220,6 +234,17 @@ ngx_stream_upstream_zone_copy_peers(ngx_slab_pool_t *shpool, } *peerp = peer; + (*peers->config)++; + } + + for (peerp = &peers->resolve; *peerp; peerp = &peer->next) { + peer = ngx_stream_upstream_zone_copy_peer(peers, *peerp); + if (peer == NULL) { + return NULL; + } + + *peerp = peer; + (*peers->config)++; } if (peers->next == NULL) { @@ -236,6 +261,7 @@ ngx_stream_upstream_zone_copy_peers(ngx_slab_pool_t *shpool, backup->name = name; backup->shpool = shpool; + backup->config = config; for (peerp = &backup->peer; *peerp; peerp = &peer->next) { /* pool is unlocked */ @@ -245,6 +271,17 @@ ngx_stream_upstream_zone_copy_peers(ngx_slab_pool_t *shpool, } *peerp = peer; + (*backup->config)++; + } + + for (peerp = &backup->resolve; *peerp; peerp = &peer->next) { + peer = ngx_stream_upstream_zone_copy_peer(backup, *peerp); + if (peer == NULL) { + return NULL; + } + + *peerp = peer; + (*backup->config)++; } peers->next = backup; @@ -276,6 +313,7 @@ ngx_stream_upstream_zone_copy_peer(ngx_stream_upstream_rr_peers_t *peers, dst->sockaddr = NULL; dst->name.data = NULL; dst->server.data = NULL; + dst->host = NULL; } dst->sockaddr = ngx_slab_calloc_locked(pool, sizeof(ngx_sockaddr_t)); @@ -298,12 +336,37 @@ ngx_stream_upstream_zone_copy_peer(ngx_stream_upstream_rr_peers_t *peers, } ngx_memcpy(dst->server.data, src->server.data, src->server.len); + + if (src->host) { + dst->host = ngx_slab_calloc_locked(pool, + sizeof(ngx_stream_upstream_host_t)); + if (dst->host == NULL) { + goto failed; + } + + dst->host->name.data = ngx_slab_alloc_locked(pool, + src->host->name.len); + if (dst->host->name.data == NULL) { + goto failed; + } + + dst->host->peers = peers; + dst->host->peer = dst; + + dst->host->name.len = src->host->name.len; + ngx_memcpy(dst->host->name.data, src->host->name.data, + src->host->name.len); + } } return dst; failed: + if (dst->host) { + ngx_slab_free_locked(pool, dst->host); + } + if (dst->server.data) { ngx_slab_free_locked(pool, dst->server.data); } @@ -320,3 +383,300 @@ ngx_stream_upstream_zone_copy_peer(ngx_stream_upstream_rr_peers_t *peers, return NULL; } + + +static void +ngx_stream_upstream_zone_set_single(ngx_stream_upstream_srv_conf_t *uscf) +{ + ngx_stream_upstream_rr_peers_t *peers; + + peers = uscf->peer.data; + + if (peers->number == 1 + && (peers->next == NULL || peers->next->number == 0)) + { + peers->single = 1; + + } else { + peers->single = 0; + } +} + + +static void +ngx_stream_upstream_zone_remove_peer_locked( + ngx_stream_upstream_rr_peers_t *peers, ngx_stream_upstream_rr_peer_t *peer) +{ + peers->total_weight -= peer->weight; + peers->number--; + peers->tries -= (peer->down == 0); + (*peers->config)++; + peers->weighted = (peers->total_weight != peers->number); + + ngx_stream_upstream_rr_peer_free(peers, peer); +} + + +static ngx_int_t +ngx_stream_upstream_zone_init_worker(ngx_cycle_t *cycle) +{ + ngx_uint_t i; + ngx_event_t *event; + ngx_stream_upstream_rr_peer_t *peer; + ngx_stream_upstream_rr_peers_t *peers; + ngx_stream_upstream_srv_conf_t *uscf, **uscfp; + ngx_stream_upstream_main_conf_t *umcf; + + if (ngx_process != NGX_PROCESS_WORKER + && ngx_process != NGX_PROCESS_SINGLE) + { + return NGX_OK; + } + + umcf = ngx_stream_cycle_get_module_main_conf(cycle, + ngx_stream_upstream_module); + + if (umcf == NULL) { + return NGX_OK; + } + + uscfp = umcf->upstreams.elts; + + for (i = 0; i < umcf->upstreams.nelts; i++) { + + uscf = uscfp[i]; + + if (uscf->shm_zone == NULL) { + continue; + } + + peers = uscf->peer.data; + + do { + ngx_stream_upstream_rr_peers_wlock(peers); + + for (peer = peers->resolve; peer; peer = peer->next) { + + if (peer->host->worker != ngx_worker) { + continue; + } + + event = &peer->host->event; + ngx_memzero(event, sizeof(ngx_event_t)); + + event->data = uscf; + event->handler = ngx_stream_upstream_zone_resolve_timer; + event->log = cycle->log; + event->cancelable = 1; + + ngx_add_timer(event, 1); + } + + ngx_stream_upstream_rr_peers_unlock(peers); + + peers = peers->next; + + } while (peers); + } + + return NGX_OK; +} + + +static void +ngx_stream_upstream_zone_resolve_timer(ngx_event_t *event) +{ + ngx_resolver_ctx_t *ctx; + ngx_stream_upstream_host_t *host; + ngx_stream_upstream_srv_conf_t *uscf; + + host = (ngx_stream_upstream_host_t *) event; + uscf = event->data; + + ctx = ngx_resolve_start(uscf->resolver, NULL); + if (ctx == NULL) { + goto retry; + } + + if (ctx == NGX_NO_RESOLVER) { + ngx_log_error(NGX_LOG_ERR, event->log, 0, + "no resolver defined to resolve %V", &host->name); + return; + } + + ctx->name = host->name; + ctx->handler = ngx_stream_upstream_zone_resolve_handler; + ctx->data = host; + ctx->timeout = uscf->resolver_timeout; + ctx->cancelable = 1; + + if (ngx_resolve_name(ctx) == NGX_OK) { + return; + } + +retry: + + ngx_add_timer(event, ngx_max(uscf->resolver_timeout, 1000)); +} + + +static void +ngx_stream_upstream_zone_resolve_handler(ngx_resolver_ctx_t *ctx) +{ + time_t now; + in_port_t port; + ngx_msec_t timer; + ngx_uint_t i, j; + ngx_event_t *event; + ngx_resolver_addr_t *addr; + ngx_stream_upstream_host_t *host; + ngx_stream_upstream_rr_peer_t *peer, *template, **peerp; + ngx_stream_upstream_rr_peers_t *peers; + ngx_stream_upstream_srv_conf_t *uscf; + + host = ctx->data; + event = &host->event; + uscf = event->data; + peers = host->peers; + template = host->peer; + + ngx_stream_upstream_rr_peers_wlock(peers); + + now = ngx_time(); + + if (ctx->state) { + ngx_log_error(NGX_LOG_ERR, event->log, 0, + "%V could not be resolved (%i: %s)", + &ctx->name, ctx->state, + ngx_resolver_strerror(ctx->state)); + + if (ctx->state != NGX_RESOLVE_NXDOMAIN) { + ngx_stream_upstream_rr_peers_unlock(peers); + + ngx_resolve_name_done(ctx); + + ngx_add_timer(event, ngx_max(uscf->resolver_timeout, 1000)); + return; + } + + /* NGX_RESOLVE_NXDOMAIN */ + + ctx->naddrs = 0; + } + +#if (NGX_DEBUG) + { + u_char text[NGX_SOCKADDR_STRLEN]; + size_t len; + + for (i = 0; i < ctx->naddrs; i++) { + len = ngx_sock_ntop(ctx->addrs[i].sockaddr, ctx->addrs[i].socklen, + text, NGX_SOCKADDR_STRLEN, 0); + + ngx_log_debug3(NGX_LOG_DEBUG_STREAM, event->log, 0, + "name %V was resolved to %*s", &host->name, len, text); + } + } +#endif + + for (peerp = &peers->peer; *peerp; /* void */ ) { + peer = *peerp; + + if (peer->host != host) { + goto next; + } + + for (j = 0; j < ctx->naddrs; j++) { + + addr = &ctx->addrs[j]; + + if (addr->name.len == 0 + && ngx_cmp_sockaddr(peer->sockaddr, peer->socklen, + addr->sockaddr, addr->socklen, 0) + == NGX_OK) + { + addr->name.len = 1; + goto next; + } + } + + *peerp = peer->next; + ngx_stream_upstream_zone_remove_peer_locked(peers, peer); + + ngx_stream_upstream_zone_set_single(uscf); + + continue; + + next: + + peerp = &peer->next; + } + + for (i = 0; i < ctx->naddrs; i++) { + + addr = &ctx->addrs[i]; + + if (addr->name.len == 1) { + addr->name.len = 0; + continue; + } + + ngx_shmtx_lock(&peers->shpool->mutex); + peer = ngx_stream_upstream_zone_copy_peer(peers, NULL); + ngx_shmtx_unlock(&peers->shpool->mutex); + if (peer == NULL) { + ngx_log_error(NGX_LOG_ERR, event->log, 0, + "cannot add new server to upstream \"%V\", " + "memory exhausted", peers->name); + break; + } + + ngx_memcpy(peer->sockaddr, addr->sockaddr, addr->socklen); + + port = ((struct sockaddr_in *) template->sockaddr)->sin_port; + + switch (peer->sockaddr->sa_family) { +#if (NGX_HAVE_INET6) + case AF_INET6: + ((struct sockaddr_in6 *) peer->sockaddr)->sin6_port = port; + break; +#endif + default: /* AF_INET */ + ((struct sockaddr_in *) peer->sockaddr)->sin_port = port; + } + + peer->socklen = addr->socklen; + + peer->name.len = ngx_sock_ntop(peer->sockaddr, peer->socklen, + peer->name.data, NGX_SOCKADDR_STRLEN, 1); + + peer->host = template->host; + peer->server = template->server; + + peer->weight = template->weight; + peer->effective_weight = peer->weight; + peer->max_conns = template->max_conns; + peer->max_fails = template->max_fails; + peer->fail_timeout = template->fail_timeout; + peer->down = template->down; + + *peerp = peer; + peerp = &peer->next; + + peers->number++; + peers->tries += (peer->down == 0); + peers->total_weight += peer->weight; + peers->weighted = (peers->total_weight != peers->number); + (*peers->config)++; + + ngx_stream_upstream_zone_set_single(uscf); + } + + ngx_stream_upstream_rr_peers_unlock(peers); + + timer = (ngx_msec_t) 1000 * (ctx->valid > now ? ctx->valid - now + 1 : 1); + + ngx_resolve_name_done(ctx); + + ngx_add_timer(event, timer); +} From 9fe119b431c957824d7bed75fce47dfbda74ca33 Mon Sep 17 00:00:00 2001 From: Dmitry Volyntsev Date: Thu, 17 Mar 2016 18:42:31 +0300 Subject: [PATCH 182/279] Upstream: construct upstream peers from DNS SRV records. --- .../modules/ngx_http_upstream_zone_module.c | 183 ++++++++++++++--- src/http/ngx_http_upstream.c | 38 ++++ src/http/ngx_http_upstream.h | 3 +- src/http/ngx_http_upstream_round_robin.c | 12 +- src/http/ngx_http_upstream_round_robin.h | 3 +- src/stream/ngx_stream_upstream.c | 45 ++++- src/stream/ngx_stream_upstream.h | 4 +- src/stream/ngx_stream_upstream_round_robin.c | 12 +- src/stream/ngx_stream_upstream_round_robin.h | 3 +- src/stream/ngx_stream_upstream_zone_module.c | 184 +++++++++++++++--- 10 files changed, 420 insertions(+), 67 deletions(-) diff --git a/src/http/modules/ngx_http_upstream_zone_module.c b/src/http/modules/ngx_http_upstream_zone_module.c index 355df39abc8..e3ede926811 100644 --- a/src/http/modules/ngx_http_upstream_zone_module.c +++ b/src/http/modules/ngx_http_upstream_zone_module.c @@ -359,6 +359,18 @@ ngx_http_upstream_zone_copy_peer(ngx_http_upstream_rr_peers_t *peers, dst->host->name.len = src->host->name.len; ngx_memcpy(dst->host->name.data, src->host->name.data, src->host->name.len); + + if (src->host->service.len) { + dst->host->service.data = ngx_slab_alloc_locked(pool, + src->host->service.len); + if (dst->host->service.data == NULL) { + goto failed; + } + + dst->host->service.len = src->host->service.len; + ngx_memcpy(dst->host->service.data, src->host->service.data, + src->host->service.len); + } } } @@ -367,6 +379,10 @@ ngx_http_upstream_zone_copy_peer(ngx_http_upstream_rr_peers_t *peers, failed: if (dst->host) { + if (dst->host->name.data) { + ngx_slab_free_locked(pool, dst->host->name.data); + } + ngx_slab_free_locked(pool, dst->host); } @@ -510,6 +526,7 @@ ngx_http_upstream_zone_resolve_timer(ngx_event_t *event) ctx->handler = ngx_http_upstream_zone_resolve_handler; ctx->data = host; ctx->timeout = uscf->resolver_timeout; + ctx->service = host->service; ctx->cancelable = 1; if (ngx_resolve_name(ctx) == NGX_OK) { @@ -522,15 +539,28 @@ ngx_http_upstream_zone_resolve_timer(ngx_event_t *event) } +#define ngx_http_upstream_zone_addr_marked(addr) \ + ((uintptr_t) (addr)->sockaddr & 1) + +#define ngx_http_upstream_zone_mark_addr(addr) \ + (addr)->sockaddr = (struct sockaddr *) ((uintptr_t) (addr)->sockaddr | 1) + +#define ngx_http_upstream_zone_unmark_addr(addr) \ + (addr)->sockaddr = \ + (struct sockaddr *) ((uintptr_t) (addr)->sockaddr & ~((uintptr_t) 1)) + static void ngx_http_upstream_zone_resolve_handler(ngx_resolver_ctx_t *ctx) { time_t now; + u_short min_priority; in_port_t port; + ngx_str_t *server; ngx_msec_t timer; - ngx_uint_t i, j; + ngx_uint_t i, j, backup, addr_backup; ngx_event_t *event; ngx_resolver_addr_t *addr; + ngx_resolver_srv_name_t *srv; ngx_http_upstream_host_t *host; ngx_http_upstream_rr_peer_t *peer, *template, **peerp; ngx_http_upstream_rr_peers_t *peers; @@ -546,11 +576,32 @@ ngx_http_upstream_zone_resolve_handler(ngx_resolver_ctx_t *ctx) now = ngx_time(); + for (i = 0; i < ctx->nsrvs; i++) { + srv = &ctx->srvs[i]; + + if (srv->state) { + ngx_log_error(NGX_LOG_ERR, event->log, 0, + "%V could not be resolved (%i: %s) " + "while resolving service %V of %V", + &srv->name, srv->state, + ngx_resolver_strerror(srv->state), &ctx->service, + &ctx->name); + } + } + if (ctx->state) { - ngx_log_error(NGX_LOG_ERR, event->log, 0, - "%V could not be resolved (%i: %s)", - &ctx->name, ctx->state, - ngx_resolver_strerror(ctx->state)); + if (ctx->service.len) { + ngx_log_error(NGX_LOG_ERR, event->log, 0, + "service %V of %V could not be resolved (%i: %s)", + &ctx->service, &ctx->name, ctx->state, + ngx_resolver_strerror(ctx->state)); + + } else { + ngx_log_error(NGX_LOG_ERR, event->log, 0, + "%V could not be resolved (%i: %s)", + &ctx->name, ctx->state, + ngx_resolver_strerror(ctx->state)); + } if (ctx->state != NGX_RESOLVE_NXDOMAIN) { ngx_http_upstream_rr_peers_unlock(peers); @@ -566,6 +617,13 @@ ngx_http_upstream_zone_resolve_handler(ngx_resolver_ctx_t *ctx) ctx->naddrs = 0; } + backup = 0; + min_priority = 65535; + + for (i = 0; i < ctx->naddrs; i++) { + min_priority = ngx_min(ctx->addrs[i].priority, min_priority); + } + #if (NGX_DEBUG) { u_char text[NGX_SOCKADDR_STRLEN]; @@ -573,14 +631,20 @@ ngx_http_upstream_zone_resolve_handler(ngx_resolver_ctx_t *ctx) for (i = 0; i < ctx->naddrs; i++) { len = ngx_sock_ntop(ctx->addrs[i].sockaddr, ctx->addrs[i].socklen, - text, NGX_SOCKADDR_STRLEN, 0); + text, NGX_SOCKADDR_STRLEN, 1); - ngx_log_debug3(NGX_LOG_DEBUG_HTTP, event->log, 0, - "name %V was resolved to %*s", &host->name, len, text); + ngx_log_debug7(NGX_LOG_DEBUG_HTTP, event->log, 0, + "name %V was resolved to %*s " + "s:\"%V\" n:\"%V\" w:%d %s", + &host->name, len, text, &host->service, + &ctx->addrs[i].name, ctx->addrs[i].weight, + ctx->addrs[i].priority != min_priority ? "backup" : ""); } } #endif +again: + for (peerp = &peers->peer; *peerp; /* void */ ) { peer = *peerp; @@ -592,14 +656,39 @@ ngx_http_upstream_zone_resolve_handler(ngx_resolver_ctx_t *ctx) addr = &ctx->addrs[j]; - if (addr->name.len == 0 - && ngx_cmp_sockaddr(peer->sockaddr, peer->socklen, - addr->sockaddr, addr->socklen, 0) - == NGX_OK) + addr_backup = (addr->priority != min_priority); + if (addr_backup != backup) { + continue; + } + + if (ngx_http_upstream_zone_addr_marked(addr)) { + continue; + } + + if (ngx_cmp_sockaddr(peer->sockaddr, peer->socklen, + addr->sockaddr, addr->socklen, + host->service.len != 0) + != NGX_OK) { - addr->name.len = 1; - goto next; + continue; + } + + if (host->service.len) { + if (addr->name.len != peer->server.len + || ngx_strncmp(addr->name.data, peer->server.data, + addr->name.len)) + { + continue; + } + + if (template->weight == 1 && addr->weight != peer->weight) { + continue; + } } + + ngx_http_upstream_zone_mark_addr(addr); + + goto next; } *peerp = peer->next; @@ -618,8 +707,13 @@ ngx_http_upstream_zone_resolve_handler(ngx_resolver_ctx_t *ctx) addr = &ctx->addrs[i]; - if (addr->name.len == 1) { - addr->name.len = 0; + addr_backup = (addr->priority != min_priority); + if (addr_backup != backup) { + continue; + } + + if (ngx_http_upstream_zone_addr_marked(addr)) { + ngx_http_upstream_zone_unmark_addr(addr); continue; } @@ -631,21 +725,14 @@ ngx_http_upstream_zone_resolve_handler(ngx_resolver_ctx_t *ctx) ngx_log_error(NGX_LOG_ERR, event->log, 0, "cannot add new server to upstream \"%V\", " "memory exhausted", peers->name); - break; + goto done; } ngx_memcpy(peer->sockaddr, addr->sockaddr, addr->socklen); - port = ((struct sockaddr_in *) template->sockaddr)->sin_port; - - switch (peer->sockaddr->sa_family) { -#if (NGX_HAVE_INET6) - case AF_INET6: - ((struct sockaddr_in6 *) peer->sockaddr)->sin6_port = port; - break; -#endif - default: /* AF_INET */ - ((struct sockaddr_in *) peer->sockaddr)->sin_port = port; + if (host->service.len == 0) { + port = ngx_inet_get_port(template->sockaddr); + ngx_inet_set_port(peer->sockaddr, port); } peer->socklen = addr->socklen; @@ -654,9 +741,30 @@ ngx_http_upstream_zone_resolve_handler(ngx_resolver_ctx_t *ctx) peer->name.data, NGX_SOCKADDR_STRLEN, 1); peer->host = template->host; - peer->server = template->server; - peer->weight = template->weight; + server = host->service.len ? &addr->name : &template->server; + + peer->server.data = ngx_slab_alloc(peers->shpool, server->len); + if (peer->server.data == NULL) { + ngx_http_upstream_rr_peer_free(peers, peer); + + ngx_log_error(NGX_LOG_ERR, event->log, 0, + "cannot add new server to upstream \"%V\", " + "memory exhausted", peers->name); + goto done; + } + + peer->server.len = server->len; + ngx_memcpy(peer->server.data, server->data, server->len); + + if (host->service.len == 0) { + peer->weight = template->weight; + + } else { + peer->weight = (template->weight != 1 ? template->weight + : addr->weight); + } + peer->effective_weight = peer->weight; peer->max_conns = template->max_conns; peer->max_fails = template->max_fails; @@ -675,8 +783,25 @@ ngx_http_upstream_zone_resolve_handler(ngx_resolver_ctx_t *ctx) ngx_http_upstream_zone_set_single(uscf); } + if (host->service.len && peers->next) { + ngx_http_upstream_rr_peers_unlock(peers); + + peers = peers->next; + backup = 1; + + ngx_http_upstream_rr_peers_wlock(peers); + + goto again; + } + +done: + ngx_http_upstream_rr_peers_unlock(peers); + while (++i < ctx->naddrs) { + ngx_http_upstream_zone_unmark_addr(&ctx->addrs[i]); + } + timer = (ngx_msec_t) 1000 * (ctx->valid > now ? ctx->valid - now + 1 : 1); ngx_resolve_name_done(ctx); diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c index efe77243913..d090d161872 100644 --- a/src/http/ngx_http_upstream.c +++ b/src/http/ngx_http_upstream.c @@ -6306,6 +6306,19 @@ ngx_http_upstream_server(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) resolve = 1; continue; } + + if (ngx_strncmp(value[i].data, "service=", 8) == 0) { + + us->service.len = value[i].len - 8; + us->service.data = &value[i].data[8]; + + if (us->service.len == 0) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "service is empty"); + return NGX_CONF_ERROR; + } + + continue; + } #endif goto invalid; @@ -6321,6 +6334,15 @@ ngx_http_upstream_server(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) /* resolve at run time */ u.no_resolve = 1; } + + if (us->service.len && !resolve) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "service upstream \"%V\" requires " + "\"resolve\" parameter", + &u.url); + return NGX_CONF_ERROR; + } + #endif if (ngx_parse_url(cf->pool, &u) != NGX_OK) { @@ -6336,6 +6358,22 @@ ngx_http_upstream_server(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) #if (NGX_HTTP_UPSTREAM_ZONE) + if (us->service.len && !u.no_port) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "service upstream \"%V\" may not have port", + &us->name); + + return NGX_CONF_ERROR; + } + + if (us->service.len && u.naddrs) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "service upstream \"%V\" requires domain name", + &us->name); + + return NGX_CONF_ERROR; + } + if (resolve && u.naddrs == 0) { ngx_addr_t *addr; diff --git a/src/http/ngx_http_upstream.h b/src/http/ngx_http_upstream.h index 0922021bde2..57ee06f8ddd 100644 --- a/src/http/ngx_http_upstream.h +++ b/src/http/ngx_http_upstream.h @@ -106,9 +106,10 @@ typedef struct { #if (NGX_HTTP_UPSTREAM_ZONE) ngx_str_t host; + ngx_str_t service; #endif - NGX_COMPAT_BEGIN(4) + NGX_COMPAT_BEGIN(2) NGX_COMPAT_END } ngx_http_upstream_server_t; diff --git a/src/http/ngx_http_upstream_round_robin.c b/src/http/ngx_http_upstream_round_robin.c index cc1b6d1a2e5..5dbd4e62662 100644 --- a/src/http/ngx_http_upstream_round_robin.c +++ b/src/http/ngx_http_upstream_round_robin.c @@ -176,6 +176,7 @@ ngx_http_upstream_init_round_robin(ngx_conf_t *cf, } peer[n].host->name = server[i].host; + peer[n].host->service = server[i].service; peer[n].sockaddr = server[i].addrs[0].sockaddr; peer[n].socklen = server[i].addrs[0].socklen; @@ -245,7 +246,15 @@ ngx_http_upstream_init_round_robin(ngx_conf_t *cf, } } - if (n + r == 0) { + if (n == 0 +#if (NGX_HTTP_UPSTREAM_ZONE) + && !resolve +#endif + ) { + return NGX_OK; + } + + if (n + r == 0 && !(us->flags & NGX_HTTP_UPSTREAM_BACKUP)) { return NGX_OK; } @@ -293,6 +302,7 @@ ngx_http_upstream_init_round_robin(ngx_conf_t *cf, } peer[n].host->name = server[i].host; + peer[n].host->service = server[i].service; peer[n].sockaddr = server[i].addrs[0].sockaddr; peer[n].socklen = server[i].addrs[0].socklen; diff --git a/src/http/ngx_http_upstream_round_robin.h b/src/http/ngx_http_upstream_round_robin.h index 06437f4059e..084b0b8869c 100644 --- a/src/http/ngx_http_upstream_round_robin.h +++ b/src/http/ngx_http_upstream_round_robin.h @@ -24,6 +24,7 @@ typedef struct { ngx_event_t event; /* must be first */ ngx_uint_t worker; ngx_str_t name; + ngx_str_t service; ngx_http_upstream_rr_peers_t *peers; ngx_http_upstream_rr_peer_t *peer; } ngx_http_upstream_host_t; @@ -150,7 +151,7 @@ ngx_http_upstream_rr_peer_free_locked(ngx_http_upstream_rr_peers_t *peers, ngx_slab_free_locked(peers->shpool, peer->sockaddr); ngx_slab_free_locked(peers->shpool, peer->name.data); - if (peer->server.data && (peer->host == NULL || peer->host->peer == peer)) { + if (peer->server.data) { ngx_slab_free_locked(peers->shpool, peer->server.data); } diff --git a/src/stream/ngx_stream_upstream.c b/src/stream/ngx_stream_upstream.c index a251cca0062..be4f13016b2 100644 --- a/src/stream/ngx_stream_upstream.c +++ b/src/stream/ngx_stream_upstream.c @@ -523,6 +523,19 @@ ngx_stream_upstream_server(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) resolve = 1; continue; } + + if (ngx_strncmp(value[i].data, "service=", 8) == 0) { + + us->service.len = value[i].len - 8; + us->service.data = &value[i].data[8]; + + if (us->service.len == 0) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "service is empty"); + return NGX_CONF_ERROR; + } + + continue; + } #endif goto invalid; @@ -537,6 +550,15 @@ ngx_stream_upstream_server(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) /* resolve at run time */ u.no_resolve = 1; } + + if (us->service.len && !resolve) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "service upstream \"%V\" requires " + "\"resolve\" parameter", + &u.url); + return NGX_CONF_ERROR; + } + #endif if (ngx_parse_url(cf->pool, &u) != NGX_OK) { @@ -548,7 +570,12 @@ ngx_stream_upstream_server(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) return NGX_CONF_ERROR; } - if (u.no_port) { + if (u.no_port +#if (NGX_STREAM_UPSTREAM_ZONE) + && us->service.len == 0 +#endif + ) + { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "no port in upstream \"%V\"", &u.url); return NGX_CONF_ERROR; @@ -558,6 +585,22 @@ ngx_stream_upstream_server(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) #if (NGX_STREAM_UPSTREAM_ZONE) + if (us->service.len && !u.no_port) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "service upstream \"%V\" may not have port", + &us->name); + + return NGX_CONF_ERROR; + } + + if (us->service.len && u.naddrs) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "service upstream \"%V\" requires domain name", + &us->name); + + return NGX_CONF_ERROR; + } + if (resolve && u.naddrs == 0) { ngx_addr_t *addr; diff --git a/src/stream/ngx_stream_upstream.h b/src/stream/ngx_stream_upstream.h index 686e9269c44..c581aa0be5f 100644 --- a/src/stream/ngx_stream_upstream.h +++ b/src/stream/ngx_stream_upstream.h @@ -65,10 +65,8 @@ typedef struct { #if (NGX_STREAM_UPSTREAM_ZONE) ngx_str_t host; + ngx_str_t service; #endif - - NGX_COMPAT_BEGIN(2) - NGX_COMPAT_END } ngx_stream_upstream_server_t; diff --git a/src/stream/ngx_stream_upstream_round_robin.c b/src/stream/ngx_stream_upstream_round_robin.c index a18171a3957..e1903b3d242 100644 --- a/src/stream/ngx_stream_upstream_round_robin.c +++ b/src/stream/ngx_stream_upstream_round_robin.c @@ -183,6 +183,7 @@ ngx_stream_upstream_init_round_robin(ngx_conf_t *cf, } peer[n].host->name = server[i].host; + peer[n].host->service = server[i].service; peer[n].sockaddr = server[i].addrs[0].sockaddr; peer[n].socklen = server[i].addrs[0].socklen; @@ -251,7 +252,15 @@ ngx_stream_upstream_init_round_robin(ngx_conf_t *cf, } } - if (n + r == 0) { + if (n == 0 +#if (NGX_STREAM_UPSTREAM_ZONE) + && !resolve +#endif + ) { + return NGX_OK; + } + + if (n + r == 0 && !(us->flags & NGX_STREAM_UPSTREAM_BACKUP)) { return NGX_OK; } @@ -299,6 +308,7 @@ ngx_stream_upstream_init_round_robin(ngx_conf_t *cf, } peer[n].host->name = server[i].host; + peer[n].host->service = server[i].service; peer[n].sockaddr = server[i].addrs[0].sockaddr; peer[n].socklen = server[i].addrs[0].socklen; diff --git a/src/stream/ngx_stream_upstream_round_robin.h b/src/stream/ngx_stream_upstream_round_robin.h index 5002200c26a..707a9889dbf 100644 --- a/src/stream/ngx_stream_upstream_round_robin.h +++ b/src/stream/ngx_stream_upstream_round_robin.h @@ -24,6 +24,7 @@ typedef struct { ngx_event_t event; /* must be first */ ngx_uint_t worker; ngx_str_t name; + ngx_str_t service; ngx_stream_upstream_rr_peers_t *peers; ngx_stream_upstream_rr_peer_t *peer; } ngx_stream_upstream_host_t; @@ -148,7 +149,7 @@ ngx_stream_upstream_rr_peer_free_locked(ngx_stream_upstream_rr_peers_t *peers, ngx_slab_free_locked(peers->shpool, peer->sockaddr); ngx_slab_free_locked(peers->shpool, peer->name.data); - if (peer->server.data && (peer->host == NULL || peer->host->peer == peer)) { + if (peer->server.data) { ngx_slab_free_locked(peers->shpool, peer->server.data); } diff --git a/src/stream/ngx_stream_upstream_zone_module.c b/src/stream/ngx_stream_upstream_zone_module.c index d731baf317c..0d6ab89f37c 100644 --- a/src/stream/ngx_stream_upstream_zone_module.c +++ b/src/stream/ngx_stream_upstream_zone_module.c @@ -356,6 +356,18 @@ ngx_stream_upstream_zone_copy_peer(ngx_stream_upstream_rr_peers_t *peers, dst->host->name.len = src->host->name.len; ngx_memcpy(dst->host->name.data, src->host->name.data, src->host->name.len); + + if (src->host->service.len) { + dst->host->service.data = ngx_slab_alloc_locked(pool, + src->host->service.len); + if (dst->host->service.data == NULL) { + goto failed; + } + + dst->host->service.len = src->host->service.len; + ngx_memcpy(dst->host->service.data, src->host->service.data, + src->host->service.len); + } } } @@ -364,6 +376,10 @@ ngx_stream_upstream_zone_copy_peer(ngx_stream_upstream_rr_peers_t *peers, failed: if (dst->host) { + if (dst->host->name.data) { + ngx_slab_free_locked(pool, dst->host->name.data); + } + ngx_slab_free_locked(pool, dst->host); } @@ -508,6 +524,7 @@ ngx_stream_upstream_zone_resolve_timer(ngx_event_t *event) ctx->handler = ngx_stream_upstream_zone_resolve_handler; ctx->data = host; ctx->timeout = uscf->resolver_timeout; + ctx->service = host->service; ctx->cancelable = 1; if (ngx_resolve_name(ctx) == NGX_OK) { @@ -520,15 +537,28 @@ ngx_stream_upstream_zone_resolve_timer(ngx_event_t *event) } +#define ngx_stream_upstream_zone_addr_marked(addr) \ + ((uintptr_t) (addr)->sockaddr & 1) + +#define ngx_stream_upstream_zone_mark_addr(addr) \ + (addr)->sockaddr = (struct sockaddr *) ((uintptr_t) (addr)->sockaddr | 1) + +#define ngx_stream_upstream_zone_unmark_addr(addr) \ + (addr)->sockaddr = \ + (struct sockaddr *) ((uintptr_t) (addr)->sockaddr & ~((uintptr_t) 1)) + static void ngx_stream_upstream_zone_resolve_handler(ngx_resolver_ctx_t *ctx) { time_t now; + u_short min_priority; in_port_t port; + ngx_str_t *server; ngx_msec_t timer; - ngx_uint_t i, j; + ngx_uint_t i, j, backup, addr_backup; ngx_event_t *event; ngx_resolver_addr_t *addr; + ngx_resolver_srv_name_t *srv; ngx_stream_upstream_host_t *host; ngx_stream_upstream_rr_peer_t *peer, *template, **peerp; ngx_stream_upstream_rr_peers_t *peers; @@ -544,11 +574,32 @@ ngx_stream_upstream_zone_resolve_handler(ngx_resolver_ctx_t *ctx) now = ngx_time(); + for (i = 0; i < ctx->nsrvs; i++) { + srv = &ctx->srvs[i]; + + if (srv->state) { + ngx_log_error(NGX_LOG_ERR, event->log, 0, + "%V could not be resolved (%i: %s) " + "while resolving service %V of %V", + &srv->name, srv->state, + ngx_resolver_strerror(srv->state), &ctx->service, + &ctx->name); + } + } + if (ctx->state) { - ngx_log_error(NGX_LOG_ERR, event->log, 0, - "%V could not be resolved (%i: %s)", - &ctx->name, ctx->state, - ngx_resolver_strerror(ctx->state)); + if (ctx->service.len) { + ngx_log_error(NGX_LOG_ERR, event->log, 0, + "service %V of %V could not be resolved (%i: %s)", + &ctx->service, &ctx->name, ctx->state, + ngx_resolver_strerror(ctx->state)); + + } else { + ngx_log_error(NGX_LOG_ERR, event->log, 0, + "%V could not be resolved (%i: %s)", + &ctx->name, ctx->state, + ngx_resolver_strerror(ctx->state)); + } if (ctx->state != NGX_RESOLVE_NXDOMAIN) { ngx_stream_upstream_rr_peers_unlock(peers); @@ -564,6 +615,13 @@ ngx_stream_upstream_zone_resolve_handler(ngx_resolver_ctx_t *ctx) ctx->naddrs = 0; } + backup = 0; + min_priority = 65535; + + for (i = 0; i < ctx->naddrs; i++) { + min_priority = ngx_min(ctx->addrs[i].priority, min_priority); + } + #if (NGX_DEBUG) { u_char text[NGX_SOCKADDR_STRLEN]; @@ -571,14 +629,20 @@ ngx_stream_upstream_zone_resolve_handler(ngx_resolver_ctx_t *ctx) for (i = 0; i < ctx->naddrs; i++) { len = ngx_sock_ntop(ctx->addrs[i].sockaddr, ctx->addrs[i].socklen, - text, NGX_SOCKADDR_STRLEN, 0); + text, NGX_SOCKADDR_STRLEN, 1); - ngx_log_debug3(NGX_LOG_DEBUG_STREAM, event->log, 0, - "name %V was resolved to %*s", &host->name, len, text); + ngx_log_debug7(NGX_LOG_DEBUG_STREAM, event->log, 0, + "name %V was resolved to %*s " + "s:\"%V\" n:\"%V\" w:%d %s", + &host->name, len, text, &host->service, + &ctx->addrs[i].name, ctx->addrs[i].weight, + ctx->addrs[i].priority != min_priority ? "backup" : ""); } } #endif +again: + for (peerp = &peers->peer; *peerp; /* void */ ) { peer = *peerp; @@ -590,14 +654,39 @@ ngx_stream_upstream_zone_resolve_handler(ngx_resolver_ctx_t *ctx) addr = &ctx->addrs[j]; - if (addr->name.len == 0 - && ngx_cmp_sockaddr(peer->sockaddr, peer->socklen, - addr->sockaddr, addr->socklen, 0) - == NGX_OK) + addr_backup = (addr->priority != min_priority); + if (addr_backup != backup) { + continue; + } + + if (ngx_stream_upstream_zone_addr_marked(addr)) { + continue; + } + + if (ngx_cmp_sockaddr(peer->sockaddr, peer->socklen, + addr->sockaddr, addr->socklen, + host->service.len != 0) + != NGX_OK) { - addr->name.len = 1; - goto next; + continue; } + + if (host->service.len) { + if (addr->name.len != peer->server.len + || ngx_strncmp(addr->name.data, peer->server.data, + addr->name.len)) + { + continue; + } + + if (template->weight == 1 && addr->weight != peer->weight) { + continue; + } + } + + ngx_stream_upstream_zone_mark_addr(addr); + + goto next; } *peerp = peer->next; @@ -616,33 +705,32 @@ ngx_stream_upstream_zone_resolve_handler(ngx_resolver_ctx_t *ctx) addr = &ctx->addrs[i]; - if (addr->name.len == 1) { - addr->name.len = 0; + addr_backup = (addr->priority != min_priority); + if (addr_backup != backup) { + continue; + } + + if (ngx_stream_upstream_zone_addr_marked(addr)) { + ngx_stream_upstream_zone_unmark_addr(addr); continue; } ngx_shmtx_lock(&peers->shpool->mutex); peer = ngx_stream_upstream_zone_copy_peer(peers, NULL); ngx_shmtx_unlock(&peers->shpool->mutex); + if (peer == NULL) { ngx_log_error(NGX_LOG_ERR, event->log, 0, "cannot add new server to upstream \"%V\", " "memory exhausted", peers->name); - break; + goto done; } ngx_memcpy(peer->sockaddr, addr->sockaddr, addr->socklen); - port = ((struct sockaddr_in *) template->sockaddr)->sin_port; - - switch (peer->sockaddr->sa_family) { -#if (NGX_HAVE_INET6) - case AF_INET6: - ((struct sockaddr_in6 *) peer->sockaddr)->sin6_port = port; - break; -#endif - default: /* AF_INET */ - ((struct sockaddr_in *) peer->sockaddr)->sin_port = port; + if (host->service.len == 0) { + port = ngx_inet_get_port(template->sockaddr); + ngx_inet_set_port(peer->sockaddr, port); } peer->socklen = addr->socklen; @@ -651,9 +739,30 @@ ngx_stream_upstream_zone_resolve_handler(ngx_resolver_ctx_t *ctx) peer->name.data, NGX_SOCKADDR_STRLEN, 1); peer->host = template->host; - peer->server = template->server; - peer->weight = template->weight; + server = host->service.len ? &addr->name : &template->server; + + peer->server.data = ngx_slab_alloc(peers->shpool, server->len); + if (peer->server.data == NULL) { + ngx_stream_upstream_rr_peer_free(peers, peer); + + ngx_log_error(NGX_LOG_ERR, event->log, 0, + "cannot add new server to upstream \"%V\", " + "memory exhausted", peers->name); + goto done; + } + + peer->server.len = server->len; + ngx_memcpy(peer->server.data, server->data, server->len); + + if (host->service.len == 0) { + peer->weight = template->weight; + + } else { + peer->weight = (template->weight != 1 ? template->weight + : addr->weight); + } + peer->effective_weight = peer->weight; peer->max_conns = template->max_conns; peer->max_fails = template->max_fails; @@ -672,8 +781,25 @@ ngx_stream_upstream_zone_resolve_handler(ngx_resolver_ctx_t *ctx) ngx_stream_upstream_zone_set_single(uscf); } + if (host->service.len && peers->next) { + ngx_stream_upstream_rr_peers_unlock(peers); + + peers = peers->next; + backup = 1; + + ngx_stream_upstream_rr_peers_wlock(peers); + + goto again; + } + +done: + ngx_stream_upstream_rr_peers_unlock(peers); + while (++i < ctx->naddrs) { + ngx_stream_upstream_zone_unmark_addr(&ctx->addrs[i]); + } + timer = (ngx_msec_t) 1000 * (ctx->valid > now ? ctx->valid - now + 1 : 1); ngx_resolve_name_done(ctx); From 1524c5e3fc9cbff6ef97ab97017a4b73bd85694b Mon Sep 17 00:00:00 2001 From: Ruslan Ermilov Date: Fri, 3 Nov 2017 22:22:21 +0300 Subject: [PATCH 183/279] Core: inheritance of non-reusable shared memory zones. When re-creating a non-reusable zone, make the pointer to the old zone available during the new zone initialization. --- src/core/ngx_cycle.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/core/ngx_cycle.c b/src/core/ngx_cycle.c index 6978c3e4347..a75bdf8786f 100644 --- a/src/core/ngx_cycle.c +++ b/src/core/ngx_cycle.c @@ -38,7 +38,7 @@ static ngx_connection_t dumb; ngx_cycle_t * ngx_init_cycle(ngx_cycle_t *old_cycle) { - void *rv; + void *rv, *data; char **senv; ngx_uint_t i, n; ngx_log_t *log; @@ -438,6 +438,8 @@ ngx_init_cycle(ngx_cycle_t *old_cycle) opart = &old_cycle->shared_memory.part; oshm_zone = opart->elts; + data = NULL; + for (n = 0; /* void */ ; n++) { if (n >= opart->nelts) { @@ -461,9 +463,13 @@ ngx_init_cycle(ngx_cycle_t *old_cycle) continue; } + if (shm_zone[i].tag == oshm_zone[n].tag && shm_zone[i].noreuse) { + data = oshm_zone[n].data; + break; + } + if (shm_zone[i].tag == oshm_zone[n].tag - && shm_zone[i].shm.size == oshm_zone[n].shm.size - && !shm_zone[i].noreuse) + && shm_zone[i].shm.size == oshm_zone[n].shm.size) { shm_zone[i].shm.addr = oshm_zone[n].shm.addr; #if (NGX_WIN32) @@ -490,7 +496,7 @@ ngx_init_cycle(ngx_cycle_t *old_cycle) goto failed; } - if (shm_zone[i].init(&shm_zone[i], NULL) != NGX_OK) { + if (shm_zone[i].init(&shm_zone[i], data) != NGX_OK) { goto failed; } From 5ebe7a4122c9653ed6b06e6577fc68904ad061c4 Mon Sep 17 00:00:00 2001 From: Ruslan Ermilov Date: Fri, 3 Nov 2017 22:22:23 +0300 Subject: [PATCH 184/279] Upstream: pre-resolve servers on reload. After configuration is reloaded, it may take some time for the re-resolvable upstream servers to resolve and become available as peers. During this time, client requests might get dropped. Such servers are now pre-resolved using the "cache" of already resolved peers from the old shared memory zone. --- .../modules/ngx_http_upstream_zone_module.c | 201 +++++++++++++++++- src/stream/ngx_stream_upstream_zone_module.c | 201 +++++++++++++++++- 2 files changed, 388 insertions(+), 14 deletions(-) diff --git a/src/http/modules/ngx_http_upstream_zone_module.c b/src/http/modules/ngx_http_upstream_zone_module.c index e3ede926811..c1931a8bb7e 100644 --- a/src/http/modules/ngx_http_upstream_zone_module.c +++ b/src/http/modules/ngx_http_upstream_zone_module.c @@ -15,9 +15,15 @@ static char *ngx_http_upstream_zone(ngx_conf_t *cf, ngx_command_t *cmd, static ngx_int_t ngx_http_upstream_init_zone(ngx_shm_zone_t *shm_zone, void *data); static ngx_http_upstream_rr_peers_t *ngx_http_upstream_zone_copy_peers( - ngx_slab_pool_t *shpool, ngx_http_upstream_srv_conf_t *uscf); + ngx_slab_pool_t *shpool, ngx_http_upstream_srv_conf_t *uscf, + ngx_http_upstream_srv_conf_t *ouscf); static ngx_http_upstream_rr_peer_t *ngx_http_upstream_zone_copy_peer( ngx_http_upstream_rr_peers_t *peers, ngx_http_upstream_rr_peer_t *src); +static ngx_int_t ngx_http_upstream_zone_preresolve( + ngx_http_upstream_rr_peer_t *resolve, + ngx_http_upstream_rr_peers_t *peers, + ngx_http_upstream_rr_peer_t *oresolve, + ngx_http_upstream_rr_peers_t *opeers); static void ngx_http_upstream_zone_set_single( ngx_http_upstream_srv_conf_t *uscf); static void ngx_http_upstream_zone_remove_peer_locked( @@ -128,11 +134,11 @@ static ngx_int_t ngx_http_upstream_init_zone(ngx_shm_zone_t *shm_zone, void *data) { size_t len; - ngx_uint_t i; + ngx_uint_t i, j; ngx_slab_pool_t *shpool; ngx_http_upstream_rr_peers_t *peers, **peersp; - ngx_http_upstream_srv_conf_t *uscf, **uscfp; - ngx_http_upstream_main_conf_t *umcf; + ngx_http_upstream_srv_conf_t *uscf, *ouscf, **uscfp, **ouscfp; + ngx_http_upstream_main_conf_t *umcf, *oumcf; shpool = (ngx_slab_pool_t *) shm_zone->shm.addr; umcf = shm_zone->data; @@ -169,6 +175,7 @@ ngx_http_upstream_init_zone(ngx_shm_zone_t *shm_zone, void *data) /* copy peers to shared memory */ peersp = (ngx_http_upstream_rr_peers_t **) (void *) &shpool->data; + oumcf = data; for (i = 0; i < umcf->upstreams.nelts; i++) { uscf = uscfp[i]; @@ -177,7 +184,38 @@ ngx_http_upstream_init_zone(ngx_shm_zone_t *shm_zone, void *data) continue; } - peers = ngx_http_upstream_zone_copy_peers(shpool, uscf); + ouscf = NULL; + + if (oumcf) { + ouscfp = oumcf->upstreams.elts; + + for (j = 0; j < oumcf->upstreams.nelts; j++) { + + if (ouscfp[j]->shm_zone == NULL) { + continue; + } + + if (ouscfp[j]->shm_zone->shm.name.len != shm_zone->shm.name.len + || ngx_memcmp(ouscfp[j]->shm_zone->shm.name.data, + shm_zone->shm.name.data, + shm_zone->shm.name.len) + != 0) + { + continue; + } + + if (ouscfp[j]->host.len == uscf->host.len + && ngx_memcmp(ouscfp[j]->host.data, uscf->host.data, + uscf->host.len) + == 0) + { + ouscf = ouscfp[j]; + break; + } + } + } + + peers = ngx_http_upstream_zone_copy_peers(shpool, uscf, ouscf); if (peers == NULL) { return NGX_ERROR; } @@ -192,12 +230,14 @@ ngx_http_upstream_init_zone(ngx_shm_zone_t *shm_zone, void *data) static ngx_http_upstream_rr_peers_t * ngx_http_upstream_zone_copy_peers(ngx_slab_pool_t *shpool, - ngx_http_upstream_srv_conf_t *uscf) + ngx_http_upstream_srv_conf_t *uscf, ngx_http_upstream_srv_conf_t *ouscf) { ngx_str_t *name; ngx_uint_t *config; ngx_http_upstream_rr_peer_t *peer, **peerp; - ngx_http_upstream_rr_peers_t *peers, *backup; + ngx_http_upstream_rr_peers_t *peers, *opeers, *backup; + + opeers = (ouscf ? ouscf->peer.data : NULL); config = ngx_slab_calloc(shpool, sizeof(ngx_uint_t)); if (config == NULL) { @@ -250,6 +290,16 @@ ngx_http_upstream_zone_copy_peers(ngx_slab_pool_t *shpool, (*peers->config)++; } + if (opeers) { + + if (ngx_http_upstream_zone_preresolve(peers->resolve, peers, + opeers->resolve, opeers) + != NGX_OK) + { + return NULL; + } + } + if (peers->next == NULL) { goto done; } @@ -289,10 +339,30 @@ ngx_http_upstream_zone_copy_peers(ngx_slab_pool_t *shpool, peers->next = backup; + if (opeers && opeers->next) { + + if (ngx_http_upstream_zone_preresolve(peers->resolve, backup, + opeers->resolve, opeers->next) + != NGX_OK) + { + return NULL; + } + + if (ngx_http_upstream_zone_preresolve(backup->resolve, backup, + opeers->next->resolve, + opeers->next) + != NGX_OK) + { + return NULL; + } + } + done: uscf->peer.data = peers; + ngx_http_upstream_zone_set_single(uscf); + return peers; } @@ -404,6 +474,123 @@ ngx_http_upstream_zone_copy_peer(ngx_http_upstream_rr_peers_t *peers, } +static ngx_int_t +ngx_http_upstream_zone_preresolve(ngx_http_upstream_rr_peer_t *resolve, + ngx_http_upstream_rr_peers_t *peers, + ngx_http_upstream_rr_peer_t *oresolve, + ngx_http_upstream_rr_peers_t *opeers) +{ + in_port_t port; + ngx_str_t *server; + ngx_http_upstream_host_t *host; + ngx_http_upstream_rr_peer_t *peer, *template, *opeer, **peerp; + + if (resolve == NULL || oresolve == NULL) { + return NGX_OK; + } + + for (peerp = &peers->peer; *peerp; peerp = &(*peerp)->next) { + /* void */ + } + + ngx_http_upstream_rr_peers_rlock(opeers); + + for (template = resolve; template; template = template->next) { + for (opeer = oresolve; opeer; opeer = opeer->next) { + + if (opeer->host->name.len != template->host->name.len + || ngx_memcmp(opeer->host->name.data, + template->host->name.data, + template->host->name.len) + != 0) + { + continue; + } + + if (opeer->host->service.len != template->host->service.len + || ngx_memcmp(opeer->host->service.data, + template->host->service.data, + template->host->service.len) + != 0) + { + continue; + } + + host = opeer->host; + + for (opeer = opeers->peer; opeer; opeer = opeer->next) { + + if (opeer->host != host) { + continue; + } + + peer = ngx_http_upstream_zone_copy_peer(peers, NULL); + if (peer == NULL) { + ngx_http_upstream_rr_peers_unlock(opeers); + return NGX_ERROR; + } + + ngx_memcpy(peer->sockaddr, opeer->sockaddr, opeer->socklen); + + if (template->host->service.len == 0) { + port = ngx_inet_get_port(template->sockaddr); + ngx_inet_set_port(peer->sockaddr, port); + } + + peer->socklen = opeer->socklen; + + peer->name.len = ngx_sock_ntop(peer->sockaddr, peer->socklen, + peer->name.data, + NGX_SOCKADDR_STRLEN, 1); + + peer->host = template->host; + + server = template->host->service.len ? &opeer->server + : &template->server; + + peer->server.data = ngx_slab_alloc(peers->shpool, server->len); + if (peer->server.data == NULL) { + ngx_http_upstream_rr_peers_unlock(opeers); + return NGX_ERROR; + } + + ngx_memcpy(peer->server.data, server->data, server->len); + peer->server.len = server->len; + + if (host->service.len == 0) { + peer->weight = template->weight; + + } else { + peer->weight = (template->weight != 1 ? template->weight + : opeer->weight); + } + + peer->effective_weight = peer->weight; + peer->max_conns = template->max_conns; + peer->max_fails = template->max_fails; + peer->fail_timeout = template->fail_timeout; + peer->down = template->down; + + (*peers->config)++; + + *peerp = peer; + peerp = &peer->next; + + peers->number++; + peers->tries += (peer->down == 0); + peers->total_weight += peer->weight; + peers->weighted = (peers->total_weight != peers->number); + } + + break; + } + } + + ngx_http_upstream_rr_peers_unlock(opeers); + return NGX_OK; +} + + static void ngx_http_upstream_zone_set_single(ngx_http_upstream_srv_conf_t *uscf) { diff --git a/src/stream/ngx_stream_upstream_zone_module.c b/src/stream/ngx_stream_upstream_zone_module.c index 0d6ab89f37c..d92609febc6 100644 --- a/src/stream/ngx_stream_upstream_zone_module.c +++ b/src/stream/ngx_stream_upstream_zone_module.c @@ -15,9 +15,15 @@ static char *ngx_stream_upstream_zone(ngx_conf_t *cf, ngx_command_t *cmd, static ngx_int_t ngx_stream_upstream_init_zone(ngx_shm_zone_t *shm_zone, void *data); static ngx_stream_upstream_rr_peers_t *ngx_stream_upstream_zone_copy_peers( - ngx_slab_pool_t *shpool, ngx_stream_upstream_srv_conf_t *uscf); + ngx_slab_pool_t *shpool, ngx_stream_upstream_srv_conf_t *uscf, + ngx_stream_upstream_srv_conf_t *ouscf); static ngx_stream_upstream_rr_peer_t *ngx_stream_upstream_zone_copy_peer( ngx_stream_upstream_rr_peers_t *peers, ngx_stream_upstream_rr_peer_t *src); +static ngx_int_t ngx_stream_upstream_zone_preresolve( + ngx_stream_upstream_rr_peer_t *resolve, + ngx_stream_upstream_rr_peers_t *peers, + ngx_stream_upstream_rr_peer_t *oresolve, + ngx_stream_upstream_rr_peers_t *opeers); static void ngx_stream_upstream_zone_set_single( ngx_stream_upstream_srv_conf_t *uscf); static void ngx_stream_upstream_zone_remove_peer_locked( @@ -125,11 +131,11 @@ static ngx_int_t ngx_stream_upstream_init_zone(ngx_shm_zone_t *shm_zone, void *data) { size_t len; - ngx_uint_t i; + ngx_uint_t i, j; ngx_slab_pool_t *shpool; ngx_stream_upstream_rr_peers_t *peers, **peersp; - ngx_stream_upstream_srv_conf_t *uscf, **uscfp; - ngx_stream_upstream_main_conf_t *umcf; + ngx_stream_upstream_srv_conf_t *uscf, *ouscf, **uscfp, **ouscfp; + ngx_stream_upstream_main_conf_t *umcf, *oumcf; shpool = (ngx_slab_pool_t *) shm_zone->shm.addr; umcf = shm_zone->data; @@ -166,6 +172,7 @@ ngx_stream_upstream_init_zone(ngx_shm_zone_t *shm_zone, void *data) /* copy peers to shared memory */ peersp = (ngx_stream_upstream_rr_peers_t **) (void *) &shpool->data; + oumcf = data; for (i = 0; i < umcf->upstreams.nelts; i++) { uscf = uscfp[i]; @@ -174,7 +181,38 @@ ngx_stream_upstream_init_zone(ngx_shm_zone_t *shm_zone, void *data) continue; } - peers = ngx_stream_upstream_zone_copy_peers(shpool, uscf); + ouscf = NULL; + + if (oumcf) { + ouscfp = oumcf->upstreams.elts; + + for (j = 0; j < oumcf->upstreams.nelts; j++) { + + if (ouscfp[j]->shm_zone == NULL) { + continue; + } + + if (ouscfp[j]->shm_zone->shm.name.len != shm_zone->shm.name.len + || ngx_memcmp(ouscfp[j]->shm_zone->shm.name.data, + shm_zone->shm.name.data, + shm_zone->shm.name.len) + != 0) + { + continue; + } + + if (ouscfp[j]->host.len == uscf->host.len + && ngx_memcmp(ouscfp[j]->host.data, uscf->host.data, + uscf->host.len) + == 0) + { + ouscf = ouscfp[j]; + break; + } + } + } + + peers = ngx_stream_upstream_zone_copy_peers(shpool, uscf, ouscf); if (peers == NULL) { return NGX_ERROR; } @@ -189,12 +227,14 @@ ngx_stream_upstream_init_zone(ngx_shm_zone_t *shm_zone, void *data) static ngx_stream_upstream_rr_peers_t * ngx_stream_upstream_zone_copy_peers(ngx_slab_pool_t *shpool, - ngx_stream_upstream_srv_conf_t *uscf) + ngx_stream_upstream_srv_conf_t *uscf, ngx_stream_upstream_srv_conf_t *ouscf) { ngx_str_t *name; ngx_uint_t *config; ngx_stream_upstream_rr_peer_t *peer, **peerp; - ngx_stream_upstream_rr_peers_t *peers, *backup; + ngx_stream_upstream_rr_peers_t *peers, *opeers, *backup; + + opeers = (ouscf ? ouscf->peer.data : NULL); config = ngx_slab_calloc(shpool, sizeof(ngx_uint_t)); if (config == NULL) { @@ -247,6 +287,16 @@ ngx_stream_upstream_zone_copy_peers(ngx_slab_pool_t *shpool, (*peers->config)++; } + if (opeers) { + + if (ngx_stream_upstream_zone_preresolve(peers->resolve, peers, + opeers->resolve, opeers) + != NGX_OK) + { + return NULL; + } + } + if (peers->next == NULL) { goto done; } @@ -286,10 +336,30 @@ ngx_stream_upstream_zone_copy_peers(ngx_slab_pool_t *shpool, peers->next = backup; + if (opeers && opeers->next) { + + if (ngx_stream_upstream_zone_preresolve(peers->resolve, backup, + opeers->resolve, opeers->next) + != NGX_OK) + { + return NULL; + } + + if (ngx_stream_upstream_zone_preresolve(backup->resolve, backup, + opeers->next->resolve, + opeers->next) + != NGX_OK) + { + return NULL; + } + } + done: uscf->peer.data = peers; + ngx_stream_upstream_zone_set_single(uscf); + return peers; } @@ -401,6 +471,123 @@ ngx_stream_upstream_zone_copy_peer(ngx_stream_upstream_rr_peers_t *peers, } +static ngx_int_t +ngx_stream_upstream_zone_preresolve(ngx_stream_upstream_rr_peer_t *resolve, + ngx_stream_upstream_rr_peers_t *peers, + ngx_stream_upstream_rr_peer_t *oresolve, + ngx_stream_upstream_rr_peers_t *opeers) +{ + in_port_t port; + ngx_str_t *server; + ngx_stream_upstream_host_t *host; + ngx_stream_upstream_rr_peer_t *peer, *template, *opeer, **peerp; + + if (resolve == NULL || oresolve == NULL) { + return NGX_OK; + } + + for (peerp = &peers->peer; *peerp; peerp = &(*peerp)->next) { + /* void */ + } + + ngx_stream_upstream_rr_peers_rlock(opeers); + + for (template = resolve; template; template = template->next) { + for (opeer = oresolve; opeer; opeer = opeer->next) { + + if (opeer->host->name.len != template->host->name.len + || ngx_memcmp(opeer->host->name.data, + template->host->name.data, + template->host->name.len) + != 0) + { + continue; + } + + if (opeer->host->service.len != template->host->service.len + || ngx_memcmp(opeer->host->service.data, + template->host->service.data, + template->host->service.len) + != 0) + { + continue; + } + + host = opeer->host; + + for (opeer = opeers->peer; opeer; opeer = opeer->next) { + + if (opeer->host != host) { + continue; + } + + peer = ngx_stream_upstream_zone_copy_peer(peers, NULL); + if (peer == NULL) { + ngx_stream_upstream_rr_peers_unlock(opeers); + return NGX_ERROR; + } + + ngx_memcpy(peer->sockaddr, opeer->sockaddr, opeer->socklen); + + if (template->host->service.len == 0) { + port = ngx_inet_get_port(template->sockaddr); + ngx_inet_set_port(peer->sockaddr, port); + } + + peer->socklen = opeer->socklen; + + peer->name.len = ngx_sock_ntop(peer->sockaddr, peer->socklen, + peer->name.data, + NGX_SOCKADDR_STRLEN, 1); + + peer->host = template->host; + + server = template->host->service.len ? &opeer->server + : &template->server; + + peer->server.data = ngx_slab_alloc(peers->shpool, server->len); + if (peer->server.data == NULL) { + ngx_stream_upstream_rr_peers_unlock(opeers); + return NGX_ERROR; + } + + ngx_memcpy(peer->server.data, server->data, server->len); + peer->server.len = server->len; + + if (host->service.len == 0) { + peer->weight = template->weight; + + } else { + peer->weight = (template->weight != 1 ? template->weight + : opeer->weight); + } + + peer->effective_weight = peer->weight; + peer->max_conns = template->max_conns; + peer->max_fails = template->max_fails; + peer->fail_timeout = template->fail_timeout; + peer->down = template->down; + + (*peers->config)++; + + *peerp = peer; + peerp = &peer->next; + + peers->number++; + peers->tries += (peer->down == 0); + peers->total_weight += peer->weight; + peers->weighted = (peers->total_weight != peers->number); + } + + break; + } + } + + ngx_stream_upstream_rr_peers_unlock(opeers); + return NGX_OK; +} + + static void ngx_stream_upstream_zone_set_single(ngx_stream_upstream_srv_conf_t *uscf) { From ea4654550ab021b5576c03b708079e3ce3e5d9ed Mon Sep 17 00:00:00 2001 From: Vladimir Homutov Date: Fri, 18 Oct 2019 16:33:15 +0300 Subject: [PATCH 185/279] Upstream: per-upstream resolver. The "resolver" and "resolver_timeout" directives can now be specified directly in the "upstream" block. --- src/http/ngx_http_upstream.c | 51 +++++++++++++++++++ src/http/ngx_http_upstream_round_robin.c | 12 ++--- src/stream/ngx_stream_upstream.c | 52 ++++++++++++++++++++ src/stream/ngx_stream_upstream_round_robin.c | 12 ++--- 4 files changed, 115 insertions(+), 12 deletions(-) diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c index d090d161872..82a2300248c 100644 --- a/src/http/ngx_http_upstream.c +++ b/src/http/ngx_http_upstream.c @@ -169,6 +169,10 @@ static ngx_int_t ngx_http_upstream_cookie_variable(ngx_http_request_t *r, static char *ngx_http_upstream(ngx_conf_t *cf, ngx_command_t *cmd, void *dummy); static char *ngx_http_upstream_server(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); +#if (NGX_HTTP_UPSTREAM_ZONE) +static char *ngx_http_upstream_resolver(ngx_conf_t *cf, ngx_command_t *cmd, + void *conf); +#endif static ngx_int_t ngx_http_upstream_set_local(ngx_http_request_t *r, ngx_http_upstream_t *u, ngx_http_upstream_local_t *local); @@ -339,6 +343,24 @@ static ngx_command_t ngx_http_upstream_commands[] = { 0, NULL }, +#if (NGX_HTTP_UPSTREAM_ZONE) + + { ngx_string("resolver"), + NGX_HTTP_UPS_CONF|NGX_CONF_1MORE, + ngx_http_upstream_resolver, + NGX_HTTP_SRV_CONF_OFFSET, + 0, + NULL }, + + { ngx_string("resolver_timeout"), + NGX_HTTP_UPS_CONF|NGX_CONF_TAKE1, + ngx_conf_set_msec_slot, + NGX_HTTP_SRV_CONF_OFFSET, + offsetof(ngx_http_upstream_srv_conf_t, resolver_timeout), + NULL }, + +#endif + ngx_null_command }; @@ -6434,6 +6456,32 @@ ngx_http_upstream_server(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) } +#if (NGX_HTTP_UPSTREAM_ZONE) + +static char * +ngx_http_upstream_resolver(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) +{ + ngx_http_upstream_srv_conf_t *uscf = conf; + + ngx_str_t *value; + + if (uscf->resolver) { + return "is duplicate"; + } + + value = cf->args->elts; + + uscf->resolver = ngx_resolver_create(cf, &value[1], cf->args->nelts - 1); + if (uscf->resolver == NULL) { + return NGX_CONF_ERROR; + } + + return NGX_CONF_OK; +} + +#endif + + ngx_http_upstream_srv_conf_t * ngx_http_upstream_add(ngx_conf_t *cf, ngx_url_t *u, ngx_uint_t flags) { @@ -6515,6 +6563,9 @@ ngx_http_upstream_add(ngx_conf_t *cf, ngx_url_t *u, ngx_uint_t flags) uscf->line = cf->conf_file->line; uscf->port = u->port; uscf->no_port = u->no_port; +#if (NGX_HTTP_UPSTREAM_ZONE) + uscf->resolver_timeout = NGX_CONF_UNSET_MSEC; +#endif if (u->naddrs == 1 && (u->port || u->family == AF_UNIX)) { uscf->servers = ngx_array_create(cf->pool, 1, diff --git a/src/http/ngx_http_upstream_round_robin.c b/src/http/ngx_http_upstream_round_robin.c index 5dbd4e62662..304494b3c37 100644 --- a/src/http/ngx_http_upstream_round_robin.c +++ b/src/http/ngx_http_upstream_round_robin.c @@ -97,15 +97,15 @@ ngx_http_upstream_init_round_robin(ngx_conf_t *cf, clcf = ngx_http_conf_get_module_loc_conf(cf, ngx_http_core_module); - us->resolver = clcf->resolver; - us->resolver_timeout = clcf->resolver_timeout; + if (us->resolver == NULL) { + us->resolver = clcf->resolver; + } /* - * Without "resolver_timeout" in http{}, the value is unset. - * Even if we set it in ngx_http_core_merge_loc_conf(), it's - * still dependent on the module order and unreliable. + * Without "resolver_timeout" in http{} the merged value is unset. */ - ngx_conf_init_msec_value(us->resolver_timeout, 30000); + ngx_conf_merge_msec_value(us->resolver_timeout, + clcf->resolver_timeout, 30000); if (resolve && (us->resolver == NULL diff --git a/src/stream/ngx_stream_upstream.c b/src/stream/ngx_stream_upstream.c index be4f13016b2..6526d3c2279 100644 --- a/src/stream/ngx_stream_upstream.c +++ b/src/stream/ngx_stream_upstream.c @@ -22,6 +22,11 @@ static char *ngx_stream_upstream(ngx_conf_t *cf, ngx_command_t *cmd, void *dummy); static char *ngx_stream_upstream_server(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); +#if (NGX_STREAM_UPSTREAM_ZONE) +static char *ngx_stream_upstream_resolver(ngx_conf_t *cf, ngx_command_t *cmd, + void *conf); +#endif + static void *ngx_stream_upstream_create_main_conf(ngx_conf_t *cf); static char *ngx_stream_upstream_init_main_conf(ngx_conf_t *cf, void *conf); @@ -42,6 +47,24 @@ static ngx_command_t ngx_stream_upstream_commands[] = { 0, NULL }, +#if (NGX_STREAM_UPSTREAM_ZONE) + + { ngx_string("resolver"), + NGX_STREAM_UPS_CONF|NGX_CONF_1MORE, + ngx_stream_upstream_resolver, + NGX_STREAM_SRV_CONF_OFFSET, + 0, + NULL }, + + { ngx_string("resolver_timeout"), + NGX_STREAM_UPS_CONF|NGX_CONF_TAKE1, + ngx_conf_set_msec_slot, + NGX_STREAM_SRV_CONF_OFFSET, + offsetof(ngx_stream_upstream_srv_conf_t, resolver_timeout), + NULL }, + +#endif + ngx_null_command }; @@ -661,6 +684,32 @@ ngx_stream_upstream_server(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) } +#if (NGX_STREAM_UPSTREAM_ZONE) + +static char * +ngx_stream_upstream_resolver(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) +{ + ngx_stream_upstream_srv_conf_t *uscf = conf; + + ngx_str_t *value; + + if (uscf->resolver) { + return "is duplicate"; + } + + value = cf->args->elts; + + uscf->resolver = ngx_resolver_create(cf, &value[1], cf->args->nelts - 1); + if (uscf->resolver == NULL) { + return NGX_CONF_ERROR; + } + + return NGX_CONF_OK; +} + +#endif + + ngx_stream_upstream_srv_conf_t * ngx_stream_upstream_add(ngx_conf_t *cf, ngx_url_t *u, ngx_uint_t flags) { @@ -739,6 +788,9 @@ ngx_stream_upstream_add(ngx_conf_t *cf, ngx_url_t *u, ngx_uint_t flags) uscf->line = cf->conf_file->line; uscf->port = u->port; uscf->no_port = u->no_port; +#if (NGX_STREAM_UPSTREAM_ZONE) + uscf->resolver_timeout = NGX_CONF_UNSET_MSEC; +#endif if (u->naddrs == 1 && (u->port || u->family == AF_UNIX)) { uscf->servers = ngx_array_create(cf->pool, 1, diff --git a/src/stream/ngx_stream_upstream_round_robin.c b/src/stream/ngx_stream_upstream_round_robin.c index e1903b3d242..5b5f20db771 100644 --- a/src/stream/ngx_stream_upstream_round_robin.c +++ b/src/stream/ngx_stream_upstream_round_robin.c @@ -104,15 +104,15 @@ ngx_stream_upstream_init_round_robin(ngx_conf_t *cf, cscf = ngx_stream_conf_get_module_srv_conf(cf, ngx_stream_core_module); - us->resolver = cscf->resolver; - us->resolver_timeout = cscf->resolver_timeout; + if (us->resolver == NULL) { + us->resolver = cscf->resolver; + } /* - * Without "resolver_timeout" in stream{}, the value is unset. - * Even if we set it in ngx_stream_core_merge_srv_conf(), it's - * still dependent on the module order and unreliable. + * Without "resolver_timeout" in stream{} the merged value is unset. */ - ngx_conf_init_msec_value(us->resolver_timeout, 30000); + ngx_conf_merge_msec_value(us->resolver_timeout, + cscf->resolver_timeout, 30000); if (resolve && (us->resolver == NULL From 29aec5720fdfc74dca8d99d5cf6dc0fcb4e4ce2f Mon Sep 17 00:00:00 2001 From: Mini Hawthorne Date: Wed, 12 Jul 2023 12:20:45 -0700 Subject: [PATCH 186/279] Upstream: copy upstream zone DNS valid time during config reload. Previously, all upstream DNS entries would be immediately re-resolved on config reload. With a large number of upstreams, this creates a spike of DNS resolution requests. These spikes can overwhelm the DNS server or cause drops on the network. This patch retains the TTL of previous resolutions across reloads by copying each upstream's name's expiry time across configuration cycles. As a result, no additional resolutions are needed. --- src/http/modules/ngx_http_upstream_zone_module.c | 12 +++++++++++- src/http/ngx_http_upstream_round_robin.h | 1 + src/stream/ngx_stream_upstream_round_robin.h | 1 + src/stream/ngx_stream_upstream_zone_module.c | 12 +++++++++++- 4 files changed, 24 insertions(+), 2 deletions(-) diff --git a/src/http/modules/ngx_http_upstream_zone_module.c b/src/http/modules/ngx_http_upstream_zone_module.c index c1931a8bb7e..2ce8a7b5b55 100644 --- a/src/http/modules/ngx_http_upstream_zone_module.c +++ b/src/http/modules/ngx_http_upstream_zone_module.c @@ -545,6 +545,8 @@ ngx_http_upstream_zone_preresolve(ngx_http_upstream_rr_peer_t *resolve, peer->host = template->host; + template->host->valid = host->valid; + server = template->host->service.len ? &opeer->server : &template->server; @@ -626,6 +628,8 @@ ngx_http_upstream_zone_remove_peer_locked(ngx_http_upstream_rr_peers_t *peers, static ngx_int_t ngx_http_upstream_zone_init_worker(ngx_cycle_t *cycle) { + time_t now; + ngx_msec_t timer; ngx_uint_t i; ngx_event_t *event; ngx_http_upstream_rr_peer_t *peer; @@ -639,6 +643,7 @@ ngx_http_upstream_zone_init_worker(ngx_cycle_t *cycle) return NGX_OK; } + now = ngx_time(); umcf = ngx_http_cycle_get_module_main_conf(cycle, ngx_http_upstream_module); if (umcf == NULL) { @@ -674,7 +679,10 @@ ngx_http_upstream_zone_init_worker(ngx_cycle_t *cycle) event->log = cycle->log; event->cancelable = 1; - ngx_add_timer(event, 1); + timer = (peer->host->valid > now) + ? (ngx_msec_t) 1000 * (peer->host->valid - now) : 1; + + ngx_add_timer(event, timer); } ngx_http_upstream_rr_peers_unlock(peers); @@ -983,6 +991,8 @@ ngx_http_upstream_zone_resolve_handler(ngx_resolver_ctx_t *ctx) done: + host->valid = ctx->valid; + ngx_http_upstream_rr_peers_unlock(peers); while (++i < ctx->naddrs) { diff --git a/src/http/ngx_http_upstream_round_robin.h b/src/http/ngx_http_upstream_round_robin.h index 084b0b8869c..2f0a51cdf2a 100644 --- a/src/http/ngx_http_upstream_round_robin.h +++ b/src/http/ngx_http_upstream_round_robin.h @@ -25,6 +25,7 @@ typedef struct { ngx_uint_t worker; ngx_str_t name; ngx_str_t service; + time_t valid; ngx_http_upstream_rr_peers_t *peers; ngx_http_upstream_rr_peer_t *peer; } ngx_http_upstream_host_t; diff --git a/src/stream/ngx_stream_upstream_round_robin.h b/src/stream/ngx_stream_upstream_round_robin.h index 707a9889dbf..c168bfe8014 100644 --- a/src/stream/ngx_stream_upstream_round_robin.h +++ b/src/stream/ngx_stream_upstream_round_robin.h @@ -25,6 +25,7 @@ typedef struct { ngx_uint_t worker; ngx_str_t name; ngx_str_t service; + time_t valid; ngx_stream_upstream_rr_peers_t *peers; ngx_stream_upstream_rr_peer_t *peer; } ngx_stream_upstream_host_t; diff --git a/src/stream/ngx_stream_upstream_zone_module.c b/src/stream/ngx_stream_upstream_zone_module.c index d92609febc6..a6874dc33d5 100644 --- a/src/stream/ngx_stream_upstream_zone_module.c +++ b/src/stream/ngx_stream_upstream_zone_module.c @@ -542,6 +542,8 @@ ngx_stream_upstream_zone_preresolve(ngx_stream_upstream_rr_peer_t *resolve, peer->host = template->host; + template->host->valid = host->valid; + server = template->host->service.len ? &opeer->server : &template->server; @@ -623,6 +625,8 @@ ngx_stream_upstream_zone_remove_peer_locked( static ngx_int_t ngx_stream_upstream_zone_init_worker(ngx_cycle_t *cycle) { + time_t now; + ngx_msec_t timer; ngx_uint_t i; ngx_event_t *event; ngx_stream_upstream_rr_peer_t *peer; @@ -636,6 +640,7 @@ ngx_stream_upstream_zone_init_worker(ngx_cycle_t *cycle) return NGX_OK; } + now = ngx_time(); umcf = ngx_stream_cycle_get_module_main_conf(cycle, ngx_stream_upstream_module); @@ -672,7 +677,10 @@ ngx_stream_upstream_zone_init_worker(ngx_cycle_t *cycle) event->log = cycle->log; event->cancelable = 1; - ngx_add_timer(event, 1); + timer = (peer->host->valid > now) + ? (ngx_msec_t) 1000 * (peer->host->valid - now) : 1; + + ngx_add_timer(event, timer); } ngx_stream_upstream_rr_peers_unlock(peers); @@ -981,6 +989,8 @@ ngx_stream_upstream_zone_resolve_handler(ngx_resolver_ctx_t *ctx) done: + host->valid = ctx->valid; + ngx_stream_upstream_rr_peers_unlock(peers); while (++i < ctx->naddrs) { From ea15896c1a5b0ce504f85c1437bae21a542cf3e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=95=AD=E6=BE=A7=E9=82=A6?= Date: Sun, 3 Nov 2024 14:36:17 +0800 Subject: [PATCH 187/279] SSL: fixed MSVC compilation after ebd18ec1812b. MSVC generates a compilation error in case #if/#endif is used in a macro parameter. --- src/http/modules/ngx_http_grpc_module.c | 8 ++++++-- src/http/modules/ngx_http_proxy_module.c | 8 ++++++-- src/http/modules/ngx_http_ssl_module.c | 8 ++++++-- src/http/modules/ngx_http_uwsgi_module.c | 8 ++++++-- src/mail/ngx_mail_ssl_module.c | 8 ++++++-- src/stream/ngx_stream_proxy_module.c | 8 ++++++-- src/stream/ngx_stream_ssl_module.c | 8 ++++++-- 7 files changed, 42 insertions(+), 14 deletions(-) diff --git a/src/http/modules/ngx_http_grpc_module.c b/src/http/modules/ngx_http_grpc_module.c index d9456843dca..0a103ac66c5 100644 --- a/src/http/modules/ngx_http_grpc_module.c +++ b/src/http/modules/ngx_http_grpc_module.c @@ -4475,12 +4475,16 @@ ngx_http_grpc_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) ngx_conf_merge_value(conf->upstream.ssl_session_reuse, prev->upstream.ssl_session_reuse, 1); +#ifndef SSL_OP_NO_TLSv1_2 ngx_conf_merge_bitmask_value(conf->ssl_protocols, prev->ssl_protocols, (NGX_CONF_BITMASK_SET -#ifndef SSL_OP_NO_TLSv1_2 |NGX_SSL_TLSv1|NGX_SSL_TLSv1_1 -#endif |NGX_SSL_TLSv1_2|NGX_SSL_TLSv1_3)); +#else + ngx_conf_merge_bitmask_value(conf->ssl_protocols, prev->ssl_protocols, + (NGX_CONF_BITMASK_SET + |NGX_SSL_TLSv1_2|NGX_SSL_TLSv1_3)); +#endif ngx_conf_merge_str_value(conf->ssl_ciphers, prev->ssl_ciphers, "DEFAULT"); diff --git a/src/http/modules/ngx_http_proxy_module.c b/src/http/modules/ngx_http_proxy_module.c index fe1952748ad..73d8ce2a89a 100644 --- a/src/http/modules/ngx_http_proxy_module.c +++ b/src/http/modules/ngx_http_proxy_module.c @@ -3942,12 +3942,16 @@ ngx_http_proxy_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) ngx_conf_merge_value(conf->upstream.ssl_session_reuse, prev->upstream.ssl_session_reuse, 1); +#ifndef SSL_OP_NO_TLSv1_2 ngx_conf_merge_bitmask_value(conf->ssl_protocols, prev->ssl_protocols, (NGX_CONF_BITMASK_SET -#ifndef SSL_OP_NO_TLSv1_2 |NGX_SSL_TLSv1|NGX_SSL_TLSv1_1 -#endif |NGX_SSL_TLSv1_2|NGX_SSL_TLSv1_3)); +#else + ngx_conf_merge_bitmask_value(conf->ssl_protocols, prev->ssl_protocols, + (NGX_CONF_BITMASK_SET + |NGX_SSL_TLSv1_2|NGX_SSL_TLSv1_3)); +#endif ngx_conf_merge_str_value(conf->ssl_ciphers, prev->ssl_ciphers, "DEFAULT"); diff --git a/src/http/modules/ngx_http_ssl_module.c b/src/http/modules/ngx_http_ssl_module.c index 1fb1e6129be..2ab9b84b714 100644 --- a/src/http/modules/ngx_http_ssl_module.c +++ b/src/http/modules/ngx_http_ssl_module.c @@ -651,12 +651,16 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) ngx_conf_merge_value(conf->early_data, prev->early_data, 0); ngx_conf_merge_value(conf->reject_handshake, prev->reject_handshake, 0); +#ifndef SSL_OP_NO_TLSv1_2 ngx_conf_merge_bitmask_value(conf->protocols, prev->protocols, (NGX_CONF_BITMASK_SET -#ifndef SSL_OP_NO_TLSv1_2 |NGX_SSL_TLSv1|NGX_SSL_TLSv1_1 -#endif |NGX_SSL_TLSv1_2|NGX_SSL_TLSv1_3)); +#else + ngx_conf_merge_bitmask_value(conf->protocols, prev->protocols, + (NGX_CONF_BITMASK_SET + |NGX_SSL_TLSv1_2|NGX_SSL_TLSv1_3)); +#endif ngx_conf_merge_size_value(conf->buffer_size, prev->buffer_size, NGX_SSL_BUFSIZE); diff --git a/src/http/modules/ngx_http_uwsgi_module.c b/src/http/modules/ngx_http_uwsgi_module.c index f2a8dbe6a08..2af5a930a1c 100644 --- a/src/http/modules/ngx_http_uwsgi_module.c +++ b/src/http/modules/ngx_http_uwsgi_module.c @@ -1877,12 +1877,16 @@ ngx_http_uwsgi_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) ngx_conf_merge_value(conf->upstream.ssl_session_reuse, prev->upstream.ssl_session_reuse, 1); +#ifndef SSL_OP_NO_TLSv1_2 ngx_conf_merge_bitmask_value(conf->ssl_protocols, prev->ssl_protocols, (NGX_CONF_BITMASK_SET -#ifndef SSL_OP_NO_TLSv1_2 |NGX_SSL_TLSv1|NGX_SSL_TLSv1_1 -#endif |NGX_SSL_TLSv1_2|NGX_SSL_TLSv1_3)); +#else + ngx_conf_merge_bitmask_value(conf->ssl_protocols, prev->ssl_protocols, + (NGX_CONF_BITMASK_SET + |NGX_SSL_TLSv1_2|NGX_SSL_TLSv1_3)); +#endif ngx_conf_merge_str_value(conf->ssl_ciphers, prev->ssl_ciphers, "DEFAULT"); diff --git a/src/mail/ngx_mail_ssl_module.c b/src/mail/ngx_mail_ssl_module.c index 2fee1adb8db..4c3f41a58db 100644 --- a/src/mail/ngx_mail_ssl_module.c +++ b/src/mail/ngx_mail_ssl_module.c @@ -343,12 +343,16 @@ ngx_mail_ssl_merge_conf(ngx_conf_t *cf, void *parent, void *child) ngx_conf_merge_value(conf->prefer_server_ciphers, prev->prefer_server_ciphers, 0); +#ifndef SSL_OP_NO_TLSv1_2 ngx_conf_merge_bitmask_value(conf->protocols, prev->protocols, (NGX_CONF_BITMASK_SET -#ifndef SSL_OP_NO_TLSv1_2 |NGX_SSL_TLSv1|NGX_SSL_TLSv1_1 -#endif |NGX_SSL_TLSv1_2|NGX_SSL_TLSv1_3)); +#else + ngx_conf_merge_bitmask_value(conf->protocols, prev->protocols, + (NGX_CONF_BITMASK_SET + |NGX_SSL_TLSv1_2|NGX_SSL_TLSv1_3)); +#endif ngx_conf_merge_uint_value(conf->verify, prev->verify, 0); ngx_conf_merge_uint_value(conf->verify_depth, prev->verify_depth, 1); diff --git a/src/stream/ngx_stream_proxy_module.c b/src/stream/ngx_stream_proxy_module.c index 0890a04aeb0..2b6f9baaf23 100644 --- a/src/stream/ngx_stream_proxy_module.c +++ b/src/stream/ngx_stream_proxy_module.c @@ -2181,12 +2181,16 @@ ngx_stream_proxy_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) ngx_conf_merge_value(conf->ssl_session_reuse, prev->ssl_session_reuse, 1); +#ifndef SSL_OP_NO_TLSv1_2 ngx_conf_merge_bitmask_value(conf->ssl_protocols, prev->ssl_protocols, (NGX_CONF_BITMASK_SET -#ifndef SSL_OP_NO_TLSv1_2 |NGX_SSL_TLSv1|NGX_SSL_TLSv1_1 -#endif |NGX_SSL_TLSv1_2|NGX_SSL_TLSv1_3)); +#else + ngx_conf_merge_bitmask_value(conf->ssl_protocols, prev->ssl_protocols, + (NGX_CONF_BITMASK_SET + |NGX_SSL_TLSv1_2|NGX_SSL_TLSv1_3)); +#endif ngx_conf_merge_str_value(conf->ssl_ciphers, prev->ssl_ciphers, "DEFAULT"); diff --git a/src/stream/ngx_stream_ssl_module.c b/src/stream/ngx_stream_ssl_module.c index 55bc54a44c9..8177d580fc8 100644 --- a/src/stream/ngx_stream_ssl_module.c +++ b/src/stream/ngx_stream_ssl_module.c @@ -882,12 +882,16 @@ ngx_stream_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) ngx_conf_merge_value(conf->reject_handshake, prev->reject_handshake, 0); +#ifndef SSL_OP_NO_TLSv1_2 ngx_conf_merge_bitmask_value(conf->protocols, prev->protocols, (NGX_CONF_BITMASK_SET -#ifndef SSL_OP_NO_TLSv1_2 |NGX_SSL_TLSv1|NGX_SSL_TLSv1_1 -#endif |NGX_SSL_TLSv1_2|NGX_SSL_TLSv1_3)); +#else + ngx_conf_merge_bitmask_value(conf->protocols, prev->protocols, + (NGX_CONF_BITMASK_SET + |NGX_SSL_TLSv1_2|NGX_SSL_TLSv1_3)); +#endif ngx_conf_merge_uint_value(conf->verify, prev->verify, 0); ngx_conf_merge_uint_value(conf->verify_depth, prev->verify_depth, 1); From a5e152b3d9addf4ae35f40ca17ab4d62bdcbe69b Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Tue, 8 Oct 2024 17:48:15 +0400 Subject: [PATCH 188/279] FastCGI: fixed create_loc_conf comments after 05b1a8f1e. --- src/http/modules/ngx_http_fastcgi_module.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/http/modules/ngx_http_fastcgi_module.c b/src/http/modules/ngx_http_fastcgi_module.c index 46a56f54e9f..743fe0c0c70 100644 --- a/src/http/modules/ngx_http_fastcgi_module.c +++ b/src/http/modules/ngx_http_fastcgi_module.c @@ -2877,7 +2877,7 @@ ngx_http_fastcgi_create_loc_conf(ngx_conf_t *cf) * conf->upstream.store_lengths = NULL; * conf->upstream.store_values = NULL; * - * conf->index.len = { 0, NULL }; + * conf->index = { 0, NULL }; */ conf->upstream.store = NGX_CONF_UNSET; From 1ac6a18585eab226277c5d41e90b8f336153c4e0 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Tue, 8 Oct 2024 18:24:00 +0400 Subject: [PATCH 189/279] SCGI: added create_loc_conf comments. --- src/http/modules/ngx_http_scgi_module.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/http/modules/ngx_http_scgi_module.c b/src/http/modules/ngx_http_scgi_module.c index f818fc4b03c..671cd2cb735 100644 --- a/src/http/modules/ngx_http_scgi_module.c +++ b/src/http/modules/ngx_http_scgi_module.c @@ -1283,6 +1283,21 @@ ngx_http_scgi_create_loc_conf(ngx_conf_t *cf) return NULL; } + /* + * set by ngx_pcalloc(): + * + * conf->upstream.bufs.num = 0; + * conf->upstream.ignore_headers = 0; + * conf->upstream.next_upstream = 0; + * conf->upstream.cache_zone = NULL; + * conf->upstream.cache_use_stale = 0; + * conf->upstream.cache_methods = 0; + * conf->upstream.temp_path = NULL; + * conf->upstream.hide_headers_hash = { NULL, 0 }; + * conf->upstream.store_lengths = NULL; + * conf->upstream.store_values = NULL; + */ + conf->upstream.store = NGX_CONF_UNSET; conf->upstream.store_access = NGX_CONF_UNSET_UINT; conf->upstream.next_upstream_tries = NGX_CONF_UNSET_UINT; From d10bf73ebad8866918e1e1002855c3a501b92e0c Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Tue, 8 Oct 2024 18:24:34 +0400 Subject: [PATCH 190/279] Uwsgi: added create_loc_conf comments. --- src/http/modules/ngx_http_uwsgi_module.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/http/modules/ngx_http_uwsgi_module.c b/src/http/modules/ngx_http_uwsgi_module.c index 2af5a930a1c..9e9682bc30d 100644 --- a/src/http/modules/ngx_http_uwsgi_module.c +++ b/src/http/modules/ngx_http_uwsgi_module.c @@ -1511,6 +1511,28 @@ ngx_http_uwsgi_create_loc_conf(ngx_conf_t *cf) return NULL; } + /* + * set by ngx_pcalloc(): + * + * conf->upstream.bufs.num = 0; + * conf->upstream.ignore_headers = 0; + * conf->upstream.next_upstream = 0; + * conf->upstream.cache_zone = NULL; + * conf->upstream.cache_use_stale = 0; + * conf->upstream.cache_methods = 0; + * conf->upstream.temp_path = NULL; + * conf->upstream.hide_headers_hash = { NULL, 0 }; + * conf->upstream.store_lengths = NULL; + * conf->upstream.store_values = NULL; + * + * conf->uwsgi_string = { 0, NULL }; + * conf->ssl = 0; + * conf->ssl_protocols = 0; + * conf->ssl_ciphers = { 0, NULL }; + * conf->ssl_trusted_certificate = { 0, NULL }; + * conf->ssl_crl = { 0, NULL }; + */ + conf->modifier1 = NGX_CONF_UNSET_UINT; conf->modifier2 = NGX_CONF_UNSET_UINT; From 36ca44f26f9e5658e880399c95969cca507dfd69 Mon Sep 17 00:00:00 2001 From: Dan Callahan Date: Fri, 6 Sep 2024 16:57:50 +0100 Subject: [PATCH 191/279] Fixed link to contributing guidelines. Absolute paths in links end up being rooted at github.com. The contributing guidelines link is broken unless we use the full URL. Also, remove superfluous "monospace formatting" for the link. --- .github/pull_request_template.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 0ee1fc3643f..14ac4639b7e 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -6,5 +6,5 @@ If this pull request addresses an issue on GitHub, make sure to reference that issue using one of the [supported keywords](https://docs.github.com/en/github/managing-your-work-on-github/linking-a-pull-request-to-an-issue). -Before creating a pull request, make sure to comply with -[`Contributing Guidelines`](/CONTRIBUTING.md). +Before creating a pull request, make sure to comply with the +[Contributing Guidelines](https://github.com/nginx/nginx/blob/master/CONTRIBUTING.md). From 7cd60cd475901016bf3e8b22b7394b136b80a0c8 Mon Sep 17 00:00:00 2001 From: Andy Pan Date: Tue, 15 Oct 2024 17:20:20 +0800 Subject: [PATCH 192/279] On DragonFly BSD 5.8+, TCP_KEEPIDLE and TCP_KEEPINTVL are in secs. --- src/os/unix/ngx_freebsd_config.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/os/unix/ngx_freebsd_config.h b/src/os/unix/ngx_freebsd_config.h index c641108b705..81534297454 100644 --- a/src/os/unix/ngx_freebsd_config.h +++ b/src/os/unix/ngx_freebsd_config.h @@ -103,7 +103,7 @@ typedef struct aiocb ngx_aiocb_t; #define NGX_LISTEN_BACKLOG -1 -#ifdef __DragonFly__ +#if (defined __DragonFly__ && __DragonFly_version < 500702) #define NGX_KEEPALIVE_FACTOR 1000 #endif From 0ebc3242d99d3b9d00891b3cddda11ff9c2e86c4 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Tue, 29 Oct 2024 00:50:40 +0400 Subject: [PATCH 193/279] SSL: error message default in object caching API. This change initializes the "err" variable, used to produce a meaningful diagnostics on error path, to a good safe value. --- src/event/ngx_event_openssl_cache.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/event/ngx_event_openssl_cache.c b/src/event/ngx_event_openssl_cache.c index f43bdb5e738..8829e28794d 100644 --- a/src/event/ngx_event_openssl_cache.c +++ b/src/event/ngx_event_openssl_cache.c @@ -138,6 +138,8 @@ ngx_ssl_cache_fetch(ngx_conf_t *cf, ngx_uint_t index, char **err, ngx_ssl_cache_type_t *type; ngx_ssl_cache_node_t *cn; + *err = NULL; + if (ngx_ssl_cache_init_key(cf->pool, index, path, &id) != NGX_OK) { return NULL; } @@ -183,6 +185,8 @@ ngx_ssl_cache_connection_fetch(ngx_pool_t *pool, ngx_uint_t index, char **err, { ngx_ssl_cache_key_t id; + *err = NULL; + if (ngx_ssl_cache_init_key(pool, index, path, &id) != NGX_OK) { return NULL; } From cb1857407bec54804191cfc5ac8173df44f0c661 Mon Sep 17 00:00:00 2001 From: Nathan Mentze Date: Tue, 19 Nov 2024 18:13:52 -0600 Subject: [PATCH 194/279] Fixed missing double quote. --- src/event/ngx_event_openssl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c index 2b1d107df53..35e9f3c887d 100644 --- a/src/event/ngx_event_openssl.c +++ b/src/event/ngx_event_openssl.c @@ -1347,7 +1347,7 @@ ngx_ssl_dhparam(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *file) if (SSL_CTX_set0_tmp_dh_pkey(ssl->ctx, dh) != 1) { ngx_ssl_error(NGX_LOG_EMERG, ssl->log, 0, - "SSL_CTX_set0_tmp_dh_pkey(\%s\") failed", file->data); + "SSL_CTX_set0_tmp_dh_pkey(\"%s\") failed", file->data); #if (OPENSSL_VERSION_NUMBER >= 0x3000001fL) EVP_PKEY_free(dh); #endif From 6ec099a3786f2ddbe007009d5526ff2ec9316d23 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Mon, 23 Sep 2024 15:51:30 +0400 Subject: [PATCH 195/279] Mp4: fixed handling an empty run of chunks in stsc atom. A specially crafted mp4 file with an empty run of chunks in the stsc atom and a large value for samples per chunk for that run, combined with a specially crafted request, allowed to store that large value in prev_samples and later in trak->end_chunk_samples while in ngx_http_mp4_crop_stsc_data(). Later in ngx_http_mp4_update_stsz_atom() this could result in buffer overread while calculating trak->end_chunk_samples_size. Now the value of samples per chunk specified for an empty run is ignored. --- src/http/modules/ngx_http_mp4_module.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/http/modules/ngx_http_mp4_module.c b/src/http/modules/ngx_http_mp4_module.c index 041ad263b56..2ca0591367d 100644 --- a/src/http/modules/ngx_http_mp4_module.c +++ b/src/http/modules/ngx_http_mp4_module.c @@ -3176,7 +3176,10 @@ ngx_http_mp4_crop_stsc_data(ngx_http_mp4_file_t *mp4, start_sample -= n; - prev_samples = samples; + if (next_chunk > chunk) { + prev_samples = samples; + } + chunk = next_chunk; samples = ngx_mp4_get_32value(entry->samples); id = ngx_mp4_get_32value(entry->id); From d1a02451c3c5767b5d0f23e138db98a9f7801335 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Wed, 2 Oct 2024 16:22:15 +0400 Subject: [PATCH 196/279] Mp4: unordered stsc chunks error for the final chunk. Currently an error is triggered if any of the chunk runs in stsc are unordered. This however does not include the final chunk run, which ends with trak->chunks + 1. The previous chunk index can be larger leading to a 32-bit overflow. This could allow to skip the validity check "if (start_sample > n)". This could later lead to a large trak->start_chunk/trak->end_chunk, which would be caught later in ngx_http_mp4_update_stco_atom() or ngx_http_mp4_update_co64_atom(). While there are no implications of the validity check being avoided, the change still adds a check to ensure the final chunk run is ordered, to produce a meaningful error and avoid a potential integer overflow. --- src/http/modules/ngx_http_mp4_module.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/http/modules/ngx_http_mp4_module.c b/src/http/modules/ngx_http_mp4_module.c index 2ca0591367d..49b0999cf2d 100644 --- a/src/http/modules/ngx_http_mp4_module.c +++ b/src/http/modules/ngx_http_mp4_module.c @@ -3189,6 +3189,13 @@ ngx_http_mp4_crop_stsc_data(ngx_http_mp4_file_t *mp4, next_chunk = trak->chunks + 1; + if (next_chunk < chunk) { + ngx_log_error(NGX_LOG_ERR, mp4->file.log, 0, + "unordered mp4 stsc chunks in \"%s\"", + mp4->file.name.data); + return NGX_ERROR; + } + ngx_log_debug4(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, "sample:%uD, chunk:%uD, chunks:%uD, samples:%uD", start_sample, chunk, next_chunk - chunk, samples); From 569948aa12409773f27572fca3d2c8e18c9c657f Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Tue, 22 Oct 2024 18:34:13 +0400 Subject: [PATCH 197/279] Mp4: prevent chunk index underflow. When cropping stsc atom, it's assumed that chunk index is never 0. Based on this assumption, start_chunk and end_chunk are calculated by subtracting 1 from it. If chunk index is zero, start_chunk or end_chunk may underflow, which will later trigger "start/end time is out mp4 stco chunks" error. The change adds an explicit check for zero chunk index to avoid underflow and report a proper error. Zero chunk index is explicitly banned in ISO/IEC 14496-12, 8.7.4 Sample To Chunk Box. It's also implicitly banned in QuickTime File Format specification. Description of chunk offset table references "Chunk 1" as the first table element. --- src/http/modules/ngx_http_mp4_module.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/http/modules/ngx_http_mp4_module.c b/src/http/modules/ngx_http_mp4_module.c index 49b0999cf2d..b7bd192dfce 100644 --- a/src/http/modules/ngx_http_mp4_module.c +++ b/src/http/modules/ngx_http_mp4_module.c @@ -3221,6 +3221,12 @@ ngx_http_mp4_crop_stsc_data(ngx_http_mp4_file_t *mp4, return NGX_ERROR; } + if (chunk == 0) { + ngx_log_error(NGX_LOG_ERR, mp4->file.log, 0, + "zero chunk in \"%s\"", mp4->file.name.data); + return NGX_ERROR; + } + target_chunk = chunk - 1; target_chunk += start_sample / samples; chunk_samples = start_sample % samples; From 476d6526b2e8297025c608425f4cad07b4f65990 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Mon, 18 Nov 2024 13:39:13 +0400 Subject: [PATCH 198/279] SSL: a new macro to set default protocol versions. This simplifies merging protocol values after ea15896 and ebd18ec. Further, as outlined in ebd18ec18, for libraries preceeding TLSv1.2+ support, only meaningful versions TLSv1 and TLSv1.1 are set by default. While here, fixed indentation. --- src/event/ngx_event_openssl.h | 7 +++++++ src/http/modules/ngx_http_grpc_module.c | 10 +--------- src/http/modules/ngx_http_proxy_module.c | 10 +--------- src/http/modules/ngx_http_ssl_module.c | 10 +--------- src/http/modules/ngx_http_uwsgi_module.c | 10 +--------- src/mail/ngx_mail_ssl_module.c | 10 +--------- src/stream/ngx_stream_proxy_module.c | 10 +--------- src/stream/ngx_stream_ssl_module.c | 10 +--------- 8 files changed, 14 insertions(+), 63 deletions(-) diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h index 6d171229c6f..2147205d610 100644 --- a/src/event/ngx_event_openssl.h +++ b/src/event/ngx_event_openssl.h @@ -187,6 +187,13 @@ typedef struct { #define NGX_SSL_TLSv1_3 0x0040 +#if (defined SSL_OP_NO_TLSv1_2 || defined SSL_OP_NO_TLSv1_3) +#define NGX_SSL_DEFAULT_PROTOCOLS (NGX_SSL_TLSv1_2|NGX_SSL_TLSv1_3) +#else +#define NGX_SSL_DEFAULT_PROTOCOLS (NGX_SSL_TLSv1|NGX_SSL_TLSv1_1) +#endif + + #define NGX_SSL_BUFFER 1 #define NGX_SSL_CLIENT 2 diff --git a/src/http/modules/ngx_http_grpc_module.c b/src/http/modules/ngx_http_grpc_module.c index 0a103ac66c5..326720447fd 100644 --- a/src/http/modules/ngx_http_grpc_module.c +++ b/src/http/modules/ngx_http_grpc_module.c @@ -4475,16 +4475,8 @@ ngx_http_grpc_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) ngx_conf_merge_value(conf->upstream.ssl_session_reuse, prev->upstream.ssl_session_reuse, 1); -#ifndef SSL_OP_NO_TLSv1_2 ngx_conf_merge_bitmask_value(conf->ssl_protocols, prev->ssl_protocols, - (NGX_CONF_BITMASK_SET - |NGX_SSL_TLSv1|NGX_SSL_TLSv1_1 - |NGX_SSL_TLSv1_2|NGX_SSL_TLSv1_3)); -#else - ngx_conf_merge_bitmask_value(conf->ssl_protocols, prev->ssl_protocols, - (NGX_CONF_BITMASK_SET - |NGX_SSL_TLSv1_2|NGX_SSL_TLSv1_3)); -#endif + (NGX_CONF_BITMASK_SET|NGX_SSL_DEFAULT_PROTOCOLS)); ngx_conf_merge_str_value(conf->ssl_ciphers, prev->ssl_ciphers, "DEFAULT"); diff --git a/src/http/modules/ngx_http_proxy_module.c b/src/http/modules/ngx_http_proxy_module.c index 73d8ce2a89a..25fa92bae32 100644 --- a/src/http/modules/ngx_http_proxy_module.c +++ b/src/http/modules/ngx_http_proxy_module.c @@ -3942,16 +3942,8 @@ ngx_http_proxy_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) ngx_conf_merge_value(conf->upstream.ssl_session_reuse, prev->upstream.ssl_session_reuse, 1); -#ifndef SSL_OP_NO_TLSv1_2 ngx_conf_merge_bitmask_value(conf->ssl_protocols, prev->ssl_protocols, - (NGX_CONF_BITMASK_SET - |NGX_SSL_TLSv1|NGX_SSL_TLSv1_1 - |NGX_SSL_TLSv1_2|NGX_SSL_TLSv1_3)); -#else - ngx_conf_merge_bitmask_value(conf->ssl_protocols, prev->ssl_protocols, - (NGX_CONF_BITMASK_SET - |NGX_SSL_TLSv1_2|NGX_SSL_TLSv1_3)); -#endif + (NGX_CONF_BITMASK_SET|NGX_SSL_DEFAULT_PROTOCOLS)); ngx_conf_merge_str_value(conf->ssl_ciphers, prev->ssl_ciphers, "DEFAULT"); diff --git a/src/http/modules/ngx_http_ssl_module.c b/src/http/modules/ngx_http_ssl_module.c index 2ab9b84b714..0e892b04d10 100644 --- a/src/http/modules/ngx_http_ssl_module.c +++ b/src/http/modules/ngx_http_ssl_module.c @@ -651,16 +651,8 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) ngx_conf_merge_value(conf->early_data, prev->early_data, 0); ngx_conf_merge_value(conf->reject_handshake, prev->reject_handshake, 0); -#ifndef SSL_OP_NO_TLSv1_2 ngx_conf_merge_bitmask_value(conf->protocols, prev->protocols, - (NGX_CONF_BITMASK_SET - |NGX_SSL_TLSv1|NGX_SSL_TLSv1_1 - |NGX_SSL_TLSv1_2|NGX_SSL_TLSv1_3)); -#else - ngx_conf_merge_bitmask_value(conf->protocols, prev->protocols, - (NGX_CONF_BITMASK_SET - |NGX_SSL_TLSv1_2|NGX_SSL_TLSv1_3)); -#endif + (NGX_CONF_BITMASK_SET|NGX_SSL_DEFAULT_PROTOCOLS)); ngx_conf_merge_size_value(conf->buffer_size, prev->buffer_size, NGX_SSL_BUFSIZE); diff --git a/src/http/modules/ngx_http_uwsgi_module.c b/src/http/modules/ngx_http_uwsgi_module.c index 9e9682bc30d..f42ae706ae3 100644 --- a/src/http/modules/ngx_http_uwsgi_module.c +++ b/src/http/modules/ngx_http_uwsgi_module.c @@ -1899,16 +1899,8 @@ ngx_http_uwsgi_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) ngx_conf_merge_value(conf->upstream.ssl_session_reuse, prev->upstream.ssl_session_reuse, 1); -#ifndef SSL_OP_NO_TLSv1_2 ngx_conf_merge_bitmask_value(conf->ssl_protocols, prev->ssl_protocols, - (NGX_CONF_BITMASK_SET - |NGX_SSL_TLSv1|NGX_SSL_TLSv1_1 - |NGX_SSL_TLSv1_2|NGX_SSL_TLSv1_3)); -#else - ngx_conf_merge_bitmask_value(conf->ssl_protocols, prev->ssl_protocols, - (NGX_CONF_BITMASK_SET - |NGX_SSL_TLSv1_2|NGX_SSL_TLSv1_3)); -#endif + (NGX_CONF_BITMASK_SET|NGX_SSL_DEFAULT_PROTOCOLS)); ngx_conf_merge_str_value(conf->ssl_ciphers, prev->ssl_ciphers, "DEFAULT"); diff --git a/src/mail/ngx_mail_ssl_module.c b/src/mail/ngx_mail_ssl_module.c index 4c3f41a58db..176e9c62472 100644 --- a/src/mail/ngx_mail_ssl_module.c +++ b/src/mail/ngx_mail_ssl_module.c @@ -343,16 +343,8 @@ ngx_mail_ssl_merge_conf(ngx_conf_t *cf, void *parent, void *child) ngx_conf_merge_value(conf->prefer_server_ciphers, prev->prefer_server_ciphers, 0); -#ifndef SSL_OP_NO_TLSv1_2 ngx_conf_merge_bitmask_value(conf->protocols, prev->protocols, - (NGX_CONF_BITMASK_SET - |NGX_SSL_TLSv1|NGX_SSL_TLSv1_1 - |NGX_SSL_TLSv1_2|NGX_SSL_TLSv1_3)); -#else - ngx_conf_merge_bitmask_value(conf->protocols, prev->protocols, - (NGX_CONF_BITMASK_SET - |NGX_SSL_TLSv1_2|NGX_SSL_TLSv1_3)); -#endif + (NGX_CONF_BITMASK_SET|NGX_SSL_DEFAULT_PROTOCOLS)); ngx_conf_merge_uint_value(conf->verify, prev->verify, 0); ngx_conf_merge_uint_value(conf->verify_depth, prev->verify_depth, 1); diff --git a/src/stream/ngx_stream_proxy_module.c b/src/stream/ngx_stream_proxy_module.c index 2b6f9baaf23..e978056ef0f 100644 --- a/src/stream/ngx_stream_proxy_module.c +++ b/src/stream/ngx_stream_proxy_module.c @@ -2181,16 +2181,8 @@ ngx_stream_proxy_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) ngx_conf_merge_value(conf->ssl_session_reuse, prev->ssl_session_reuse, 1); -#ifndef SSL_OP_NO_TLSv1_2 ngx_conf_merge_bitmask_value(conf->ssl_protocols, prev->ssl_protocols, - (NGX_CONF_BITMASK_SET - |NGX_SSL_TLSv1|NGX_SSL_TLSv1_1 - |NGX_SSL_TLSv1_2|NGX_SSL_TLSv1_3)); -#else - ngx_conf_merge_bitmask_value(conf->ssl_protocols, prev->ssl_protocols, - (NGX_CONF_BITMASK_SET - |NGX_SSL_TLSv1_2|NGX_SSL_TLSv1_3)); -#endif + (NGX_CONF_BITMASK_SET|NGX_SSL_DEFAULT_PROTOCOLS)); ngx_conf_merge_str_value(conf->ssl_ciphers, prev->ssl_ciphers, "DEFAULT"); diff --git a/src/stream/ngx_stream_ssl_module.c b/src/stream/ngx_stream_ssl_module.c index 8177d580fc8..dfbaa0e2f6a 100644 --- a/src/stream/ngx_stream_ssl_module.c +++ b/src/stream/ngx_stream_ssl_module.c @@ -882,16 +882,8 @@ ngx_stream_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) ngx_conf_merge_value(conf->reject_handshake, prev->reject_handshake, 0); -#ifndef SSL_OP_NO_TLSv1_2 ngx_conf_merge_bitmask_value(conf->protocols, prev->protocols, - (NGX_CONF_BITMASK_SET - |NGX_SSL_TLSv1|NGX_SSL_TLSv1_1 - |NGX_SSL_TLSv1_2|NGX_SSL_TLSv1_3)); -#else - ngx_conf_merge_bitmask_value(conf->protocols, prev->protocols, - (NGX_CONF_BITMASK_SET - |NGX_SSL_TLSv1_2|NGX_SSL_TLSv1_3)); -#endif + (NGX_CONF_BITMASK_SET|NGX_SSL_DEFAULT_PROTOCOLS)); ngx_conf_merge_uint_value(conf->verify, prev->verify, 0); ngx_conf_merge_uint_value(conf->verify_depth, prev->verify_depth, 1); From 0864cca4d74e215acdcab20a68e025c6e3ee9efa Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Fri, 22 Nov 2024 11:38:06 +0400 Subject: [PATCH 199/279] QUIC: prevented BIO leak in case of error. --- src/event/quic/ngx_event_quic_openssl_compat.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/event/quic/ngx_event_quic_openssl_compat.c b/src/event/quic/ngx_event_quic_openssl_compat.c index c7412e82be3..6052bc683ba 100644 --- a/src/event/quic/ngx_event_quic_openssl_compat.c +++ b/src/event/quic/ngx_event_quic_openssl_compat.c @@ -391,6 +391,7 @@ SSL_set_quic_method(SSL *ssl, const SSL_QUIC_METHOD *quic_method) wbio = BIO_new(BIO_s_null()); if (wbio == NULL) { + BIO_free(rbio); return 0; } From a448dd52ee27ec3a550cb7d03fd27153f4799f0c Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Thu, 21 Nov 2024 12:35:50 +0400 Subject: [PATCH 200/279] Upstream: disallow empty path in proxy_store and friends. Renaming a temporary file to an empty path ("") returns NGX_ENOPATH with a subsequent ngx_create_full_path() to create the full path. This function skips initial bytes as part of path separator lookup, which causes out of bounds access on short strings. The fix is to avoid renaming a temporary file to an obviously invalid path, as well as explicitly forbid such syntax for literal values. Although Coverity reports about potential type underflow, it is not actually possible because the terminating '\0' is always included. Notably, the run-time check is sufficient enough for Win32 as well. Other short invalid values result either in NGX_ENOENT or NGX_EEXIST and "MoveFile() .. failed" critical log messages, which involves a separate error handling. Prodded by Coverity (CID 1605485). --- src/http/modules/ngx_http_fastcgi_module.c | 5 +++++ src/http/modules/ngx_http_proxy_module.c | 5 +++++ src/http/modules/ngx_http_scgi_module.c | 5 +++++ src/http/modules/ngx_http_uwsgi_module.c | 5 +++++ src/http/ngx_http_upstream.c | 4 ++++ 5 files changed, 24 insertions(+) diff --git a/src/http/modules/ngx_http_fastcgi_module.c b/src/http/modules/ngx_http_fastcgi_module.c index 743fe0c0c70..a41b496dc8f 100644 --- a/src/http/modules/ngx_http_fastcgi_module.c +++ b/src/http/modules/ngx_http_fastcgi_module.c @@ -3781,6 +3781,11 @@ ngx_http_fastcgi_store(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) return NGX_CONF_OK; } + if (value[1].len == 0) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "empty path"); + return NGX_CONF_ERROR; + } + #if (NGX_HTTP_CACHE) if (flcf->upstream.cache > 0) { return "is incompatible with \"fastcgi_cache\""; diff --git a/src/http/modules/ngx_http_proxy_module.c b/src/http/modules/ngx_http_proxy_module.c index 25fa92bae32..855ef523dd8 100644 --- a/src/http/modules/ngx_http_proxy_module.c +++ b/src/http/modules/ngx_http_proxy_module.c @@ -4943,6 +4943,11 @@ ngx_http_proxy_store(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) return NGX_CONF_OK; } + if (value[1].len == 0) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "empty path"); + return NGX_CONF_ERROR; + } + #if (NGX_HTTP_CACHE) if (plcf->upstream.cache > 0) { return "is incompatible with \"proxy_cache\""; diff --git a/src/http/modules/ngx_http_scgi_module.c b/src/http/modules/ngx_http_scgi_module.c index 671cd2cb735..9023a36e49c 100644 --- a/src/http/modules/ngx_http_scgi_module.c +++ b/src/http/modules/ngx_http_scgi_module.c @@ -1995,6 +1995,11 @@ ngx_http_scgi_store(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) return NGX_CONF_OK; } + if (value[1].len == 0) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "empty path"); + return NGX_CONF_ERROR; + } + #if (NGX_HTTP_CACHE) if (scf->upstream.cache > 0) { return "is incompatible with \"scgi_cache\""; diff --git a/src/http/modules/ngx_http_uwsgi_module.c b/src/http/modules/ngx_http_uwsgi_module.c index f42ae706ae3..7988cc589e2 100644 --- a/src/http/modules/ngx_http_uwsgi_module.c +++ b/src/http/modules/ngx_http_uwsgi_module.c @@ -2322,6 +2322,11 @@ ngx_http_uwsgi_store(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) return NGX_CONF_OK; } + if (value[1].len == 0) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "empty path"); + return NGX_CONF_ERROR; + } + #if (NGX_HTTP_CACHE) if (uwcf->upstream.cache > 0) { diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c index 82a2300248c..d95662c5661 100644 --- a/src/http/ngx_http_upstream.c +++ b/src/http/ngx_http_upstream.c @@ -4357,6 +4357,10 @@ ngx_http_upstream_store(ngx_http_request_t *r, ngx_http_upstream_t *u) "upstream stores \"%s\" to \"%s\"", tf->file.name.data, path.data); + if (path.len == 0) { + return; + } + (void) ngx_ext_rename_file(&tf->file.name, &path, &ext); u->store = 0; From 9a025219f661fbe2148659cad490c06d5e3283df Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Mon, 7 Oct 2024 18:19:24 +0400 Subject: [PATCH 201/279] QUIC: constified nonce parameter of crypto functions. This follows OpenSSL and BoringSSL API, and gives a hint to compiler that this parameter may not be modified. --- src/event/quic/ngx_event_quic_protection.c | 12 ++++++------ src/event/quic/ngx_event_quic_protection.h | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/event/quic/ngx_event_quic_protection.c b/src/event/quic/ngx_event_quic_protection.c index 55f0f6fd70f..57492825d8c 100644 --- a/src/event/quic/ngx_event_quic_protection.c +++ b/src/event/quic/ngx_event_quic_protection.c @@ -29,10 +29,10 @@ static uint64_t ngx_quic_parse_pn(u_char **pos, ngx_int_t len, u_char *mask, uint64_t *largest_pn); static ngx_int_t ngx_quic_crypto_open(ngx_quic_secret_t *s, ngx_str_t *out, - u_char *nonce, ngx_str_t *in, ngx_str_t *ad, ngx_log_t *log); + const u_char *nonce, ngx_str_t *in, ngx_str_t *ad, ngx_log_t *log); #ifndef OPENSSL_IS_BORINGSSL static ngx_int_t ngx_quic_crypto_common(ngx_quic_secret_t *s, ngx_str_t *out, - u_char *nonce, ngx_str_t *in, ngx_str_t *ad, ngx_log_t *log); + const u_char *nonce, ngx_str_t *in, ngx_str_t *ad, ngx_log_t *log); #endif static ngx_int_t ngx_quic_crypto_hp_init(const EVP_CIPHER *cipher, @@ -441,7 +441,7 @@ ngx_quic_crypto_init(const ngx_quic_cipher_t *cipher, ngx_quic_secret_t *s, static ngx_int_t -ngx_quic_crypto_open(ngx_quic_secret_t *s, ngx_str_t *out, u_char *nonce, +ngx_quic_crypto_open(ngx_quic_secret_t *s, ngx_str_t *out, const u_char *nonce, ngx_str_t *in, ngx_str_t *ad, ngx_log_t *log) { #ifdef OPENSSL_IS_BORINGSSL @@ -461,7 +461,7 @@ ngx_quic_crypto_open(ngx_quic_secret_t *s, ngx_str_t *out, u_char *nonce, ngx_int_t -ngx_quic_crypto_seal(ngx_quic_secret_t *s, ngx_str_t *out, u_char *nonce, +ngx_quic_crypto_seal(ngx_quic_secret_t *s, ngx_str_t *out, const u_char *nonce, ngx_str_t *in, ngx_str_t *ad, ngx_log_t *log) { #ifdef OPENSSL_IS_BORINGSSL @@ -483,8 +483,8 @@ ngx_quic_crypto_seal(ngx_quic_secret_t *s, ngx_str_t *out, u_char *nonce, #ifndef OPENSSL_IS_BORINGSSL static ngx_int_t -ngx_quic_crypto_common(ngx_quic_secret_t *s, ngx_str_t *out, u_char *nonce, - ngx_str_t *in, ngx_str_t *ad, ngx_log_t *log) +ngx_quic_crypto_common(ngx_quic_secret_t *s, ngx_str_t *out, + const u_char *nonce, ngx_str_t *in, ngx_str_t *ad, ngx_log_t *log) { int len, enc; ngx_quic_crypto_ctx_t *ctx; diff --git a/src/event/quic/ngx_event_quic_protection.h b/src/event/quic/ngx_event_quic_protection.h index 34cfee61be2..c09456f53cd 100644 --- a/src/event/quic/ngx_event_quic_protection.h +++ b/src/event/quic/ngx_event_quic_protection.h @@ -111,7 +111,7 @@ ngx_int_t ngx_quic_ciphers(ngx_uint_t id, ngx_quic_ciphers_t *ciphers); ngx_int_t ngx_quic_crypto_init(const ngx_quic_cipher_t *cipher, ngx_quic_secret_t *s, ngx_quic_md_t *key, ngx_int_t enc, ngx_log_t *log); ngx_int_t ngx_quic_crypto_seal(ngx_quic_secret_t *s, ngx_str_t *out, - u_char *nonce, ngx_str_t *in, ngx_str_t *ad, ngx_log_t *log); + const u_char *nonce, ngx_str_t *in, ngx_str_t *ad, ngx_log_t *log); void ngx_quic_crypto_cleanup(ngx_quic_secret_t *s); ngx_int_t ngx_quic_hkdf_expand(ngx_quic_hkdf_t *hkdf, const EVP_MD *digest, ngx_log_t *log); From 3f755b5a9e7145d5ce6b897d2298d5f6c544acf7 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Mon, 7 Oct 2024 18:43:48 +0400 Subject: [PATCH 202/279] QUIC: got rid of memory copy when initializing constant values. --- src/event/quic/ngx_event_quic_protection.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/event/quic/ngx_event_quic_protection.c b/src/event/quic/ngx_event_quic_protection.c index 57492825d8c..3f249b36aa2 100644 --- a/src/event/quic/ngx_event_quic_protection.c +++ b/src/event/quic/ngx_event_quic_protection.c @@ -18,6 +18,9 @@ #define NGX_QUIC_INITIAL_CIPHER TLS1_3_CK_AES_128_GCM_SHA256 +#define ngx_quic_md(str) { sizeof(str) - 1, str } + + static ngx_int_t ngx_hkdf_expand(u_char *out_key, size_t out_len, const EVP_MD *digest, const u_char *prk, size_t prk_len, const u_char *info, size_t info_len); @@ -606,7 +609,8 @@ ngx_quic_crypto_hp(ngx_quic_secret_t *s, u_char *out, u_char *in, { int outlen; EVP_CIPHER_CTX *ctx; - u_char zero[NGX_QUIC_HP_LEN] = {0}; + + static const u_char zero[NGX_QUIC_HP_LEN]; ctx = s->hp_ctx; @@ -948,16 +952,15 @@ ngx_quic_create_retry_packet(ngx_quic_header_t *pkt, ngx_str_t *res) { u_char *start; ngx_str_t ad, itag; - ngx_quic_md_t key; ngx_quic_secret_t secret; ngx_quic_ciphers_t ciphers; /* 5.8. Retry Packet Integrity */ - static u_char key_data[16] = - "\xbe\x0c\x69\x0b\x9f\x66\x57\x5a\x1d\x76\x6b\x54\xe3\x68\xc8\x4e"; - static u_char nonce[NGX_QUIC_IV_LEN] = + static ngx_quic_md_t key = ngx_quic_md( + "\xbe\x0c\x69\x0b\x9f\x66\x57\x5a\x1d\x76\x6b\x54\xe3\x68\xc8\x4e"); + static const u_char nonce[NGX_QUIC_IV_LEN] = "\x46\x15\x99\xd3\x5d\x63\x2b\xf2\x23\x98\x25\xbb"; - static ngx_str_t in = ngx_string(""); + static ngx_str_t in = ngx_string(""); ad.data = res->data; ad.len = ngx_quic_create_retry_itag(pkt, ad.data, &start); @@ -974,8 +977,6 @@ ngx_quic_create_retry_packet(ngx_quic_header_t *pkt, ngx_str_t *res) return NGX_ERROR; } - key.len = sizeof(key_data); - ngx_memcpy(key.data, key_data, sizeof(key_data)); secret.iv.len = NGX_QUIC_IV_LEN; if (ngx_quic_crypto_init(ciphers.c, &secret, &key, 1, pkt->log) From b2a67d261496555a46b8931935bf822ce9938294 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Mon, 11 Nov 2024 22:28:30 +0400 Subject: [PATCH 203/279] Realip: allowed square brackets with portless IPv6 address. When client address is received, IPv6 address could be specified without square brackets and without port, as well as both with the brackets and port. The change allows IPv6 in square brackets and no port, which was previously considered an error. This format conforms to RFC 3986. The change also affects proxy_bind and friends. --- src/core/ngx_inet.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/core/ngx_inet.c b/src/core/ngx_inet.c index acb2ef48a1e..2233e617bb4 100644 --- a/src/core/ngx_inet.c +++ b/src/core/ngx_inet.c @@ -639,7 +639,11 @@ ngx_parse_addr_port(ngx_pool_t *pool, ngx_addr_t *addr, u_char *text, p = ngx_strlchr(text, last, ']'); - if (p == NULL || p == last - 1 || *++p != ':') { + if (p == last - 1) { + return ngx_parse_addr(pool, addr, text + 1, len - 2); + } + + if (p == NULL || *++p != ':') { return NGX_DECLINED; } From ce88b171236de50843dba2c427a8b3e42778f2ca Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Thu, 24 Oct 2024 00:52:21 +0400 Subject: [PATCH 204/279] Mail: handling of LOGIN IMAP command untagged response. In particular, an untagged CAPABILITY response as described in the interim RFC 3501 internet drafts was seen in various IMAP servers. Previously resulted in a broken connection, now an untagged response is proxied to client. --- src/mail/ngx_mail_proxy_module.c | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/mail/ngx_mail_proxy_module.c b/src/mail/ngx_mail_proxy_module.c index efed9ab3e61..1c6d0372e74 100644 --- a/src/mail/ngx_mail_proxy_module.c +++ b/src/mail/ngx_mail_proxy_module.c @@ -1019,12 +1019,36 @@ ngx_mail_proxy_read_response(ngx_mail_session_t *s, ngx_uint_t state) break; case ngx_imap_passwd: + + /* + * untagged CAPABILITY response (draft-crispin-imapv-16), + * known to be sent by SmarterMail and Gmail + */ + + if (p[0] == '*' && p[1] == ' ') { + p += 2; + + while (p < b->last - 1) { + if (p[0] == CR && p[1] == LF) { + p += 2; + break; + } + + p++; + } + + if (b->last - p < 4) { + return NGX_AGAIN; + } + } + if (ngx_strncmp(p, s->tag.data, s->tag.len) == 0) { p += s->tag.len; if (p[0] == 'O' && p[1] == 'K') { return NGX_OK; } } + break; } From e7bd2557458c26839da89e694067017eeb214348 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Tue, 26 Nov 2024 15:36:52 +0400 Subject: [PATCH 205/279] nginx-1.27.3-RELEASE --- docs/xml/nginx/changes.xml | 93 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/docs/xml/nginx/changes.xml b/docs/xml/nginx/changes.xml index c18dedff43c..b469617042c 100644 --- a/docs/xml/nginx/changes.xml +++ b/docs/xml/nginx/changes.xml @@ -5,6 +5,99 @@ + + + + +директива server в блоке upstream поддерживает +параметр resolve. + + +the "server" directive in the "upstream" block supports +the "resolve" parameter. + + + + + +директивы resolver и resolver_timeout в блоке upstream. + + +the "resolver" and "resolver_timeout" directives in the "upstream" block. + + + + + +поддержка SmarterMail-специфичного режима +IMAP LOGIN с нетегированным ответом CAPABILITY +в почтовом прокси-сервере. + + +SmarterMail specific mode support +for IMAP LOGIN with untagged CAPABILITY response +in the mail proxy module. + + + + + +теперь протоколы TLSv1 и TLSv1.1 по умолчанию запрещены. + + +now TLSv1 and TLSv1.1 protocols are disabled by default. + + + + + +IPv6-адрес в квадратных скобках без порта теперь можно указывать +в директивах proxy_bind, fastcgi_bind, grpc_bind, memcached_bind, +scgi_bind и uwsgi_bind, +а также как адрес клиента в модуле ngx_http_realip_module. + + +an IPv6 address in square brackets and no port can be specified +in the "proxy_bind", "fastcgi_bind", "grpc_bind", "memcached_bind", +"scgi_bind", and "uwsgi_bind" directives, +and as client address in ngx_http_realip_module. + + + + + +в модуле ngx_http_mp4_module.
+Спасибо Nils Bars. + + +in the ngx_http_mp4_module.
+Thanks to Nils Bars. + + + + + +параметр so_keepalive директивы listen +мог работать некорректно на DragonFly BSD. + + +the "so_keepalive" parameter of the "listen" directive +might be handled incorrectly on DragonFly BSD. + + + + + +в директиве proxy_store. + + +in the "proxy_store" directive. + + + + + + From e28ef42b97d5352e514f019cd3b9a6c07a87c580 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Wed, 27 Nov 2024 20:01:26 +0400 Subject: [PATCH 206/279] Version bump. --- src/core/nginx.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/nginx.h b/src/core/nginx.h index fb6fa47370a..29f2c949f28 100644 --- a/src/core/nginx.h +++ b/src/core/nginx.h @@ -9,8 +9,8 @@ #define _NGINX_H_INCLUDED_ -#define nginx_version 1027003 -#define NGINX_VERSION "1.27.3" +#define nginx_version 1027004 +#define NGINX_VERSION "1.27.4" #define NGINX_VER "nginx/" NGINX_VERSION #ifdef NGX_BUILD From 930caed3bfc84e43bf4bd034150c17604dc5dc73 Mon Sep 17 00:00:00 2001 From: nandsky Date: Mon, 25 Nov 2024 15:26:29 +0800 Subject: [PATCH 207/279] QUIC: fixed client request timeout in 0-RTT scenarios. Since 0-RTT and 1-RTT data exist in the same packet number space, ngx_quic_discard_ctx incorrectly discards 1-RTT packets when 0-RTT keys are discarded. The issue was introduced by 58b92177e7c3c50f77f807ab3846ad5c7bbf0ebe. --- src/event/quic/ngx_event_quic.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/event/quic/ngx_event_quic.c b/src/event/quic/ngx_event_quic.c index c03b1d003ad..308597e27ba 100644 --- a/src/event/quic/ngx_event_quic.c +++ b/src/event/quic/ngx_event_quic.c @@ -1029,7 +1029,7 @@ ngx_quic_handle_payload(ngx_connection_t *c, ngx_quic_header_t *pkt) * After receiving a 1-RTT packet, servers MUST discard * 0-RTT keys within a short time */ - ngx_quic_discard_ctx(c, ssl_encryption_early_data); + ngx_quic_keys_discard(qc->keys, ssl_encryption_early_data); } if (qc->closing) { From c73fb273acc31bff7c4e469efda5f3fd66c48557 Mon Sep 17 00:00:00 2001 From: Jordan Zebor Date: Mon, 23 Dec 2024 08:07:01 -0800 Subject: [PATCH 208/279] Updated security policy to clarify experimental features. The original security policy language did not capture the scope as intended for experimental features and availability. --- SECURITY.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index f5cfcd7885f..8e173ed165a 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -69,8 +69,7 @@ limiting, or buffer size configurations, or applying changes is impractical. Availability issues excluded from the security release process: - Local file content or upstream response content resulting only in worker process termination. -- Issues with experimental features which result only in worker process -termination. +- Issues with experimental features which result only in availability impact. ## Trusted Configurations and Misconfigurations From a52ba8ba0e349585e49073c168e423c12abcf597 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Fri, 13 Dec 2024 13:25:26 +0400 Subject: [PATCH 209/279] QUIC: ignore version negotiation packets. Previously, such packets were treated as long header packets with unknown version 0, and a version negotiation packet was sent in response. This could be used to set up an infinite traffic reflect loop with another nginx instance. Now version negotiation packets are ignored. As per RFC 9000, Section 6.1: An endpoint MUST NOT send a Version Negotiation packet in response to receiving a Version Negotiation packet. --- src/event/quic/ngx_event_quic_transport.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/event/quic/ngx_event_quic_transport.c b/src/event/quic/ngx_event_quic_transport.c index fba098caa95..bb13447b55f 100644 --- a/src/event/quic/ngx_event_quic_transport.c +++ b/src/event/quic/ngx_event_quic_transport.c @@ -295,6 +295,11 @@ ngx_quic_parse_packet(ngx_quic_header_t *pkt) return NGX_ERROR; } + if (pkt->version == 0) { + /* version negotiation */ + return NGX_ERROR; + } + if (!ngx_quic_supported_version(pkt->version)) { return NGX_ABORT; } From e3a9b6ad08a86e799a3d77da3f2fc507d3c9699e Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Tue, 10 Dec 2024 18:19:27 +0400 Subject: [PATCH 210/279] QUIC: fixed accessing a released stream. While trying to close a stream in ngx_quic_close_streams() by calling its read event handler, the next stream saved prior to that could be destroyed recursively. This caused a segfault while trying to access the next stream. The way the next stream could be destroyed in HTTP/3 is the following. A request stream read event handler ngx_http_request_handler() could end up calling ngx_http_v3_send_cancel_stream() to report a cancelled request stream in the decoder stream. If sending stream cancellation decoder instruction fails for any reason, and the decoder stream is the next in order after the request stream, the issue is triggered. The fix is to postpone calling read event handlers for all streams being closed to avoid closing a released stream. --- src/event/quic/ngx_event_quic_streams.c | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/event/quic/ngx_event_quic_streams.c b/src/event/quic/ngx_event_quic_streams.c index 178b805e450..a9a21f5785f 100644 --- a/src/event/quic/ngx_event_quic_streams.c +++ b/src/event/quic/ngx_event_quic_streams.c @@ -174,7 +174,7 @@ ngx_int_t ngx_quic_close_streams(ngx_connection_t *c, ngx_quic_connection_t *qc) { ngx_pool_t *pool; - ngx_queue_t *q; + ngx_queue_t *q, posted_events; ngx_rbtree_t *tree; ngx_connection_t *sc; ngx_rbtree_node_t *node; @@ -197,6 +197,8 @@ ngx_quic_close_streams(ngx_connection_t *c, ngx_quic_connection_t *qc) return NGX_OK; } + ngx_queue_init(&posted_events); + node = ngx_rbtree_min(tree->root, tree->sentinel); while (node) { @@ -213,15 +215,21 @@ ngx_quic_close_streams(ngx_connection_t *c, ngx_quic_connection_t *qc) } sc->read->error = 1; + sc->read->ready = 1; sc->write->error = 1; - - ngx_quic_set_event(sc->read); - ngx_quic_set_event(sc->write); + sc->write->ready = 1; sc->close = 1; - sc->read->handler(sc->read); + + if (sc->read->posted) { + ngx_delete_posted_event(sc->read); + } + + ngx_post_event(sc->read, &posted_events); } + ngx_event_process_posted((ngx_cycle_t *) ngx_cycle, &posted_events); + if (tree->root == tree->sentinel) { return NGX_OK; } From febe6e728ff83cfc5d5bcc0c74b4d8d63dc296b0 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Thu, 9 Jan 2025 17:00:14 +0400 Subject: [PATCH 211/279] Year 2025. --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index a7ec58a754e..4efd90b510a 100644 --- a/LICENSE +++ b/LICENSE @@ -1,5 +1,5 @@ Copyright (C) 2002-2021 Igor Sysoev -Copyright (C) 2011-2024 Nginx, Inc. +Copyright (C) 2011-2025 Nginx, Inc. All rights reserved. Redistribution and use in source and binary forms, with or without From 57d54fd922e7ecbebb78598d13adc9df1a4b69c0 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Mon, 23 Dec 2024 17:57:45 +0400 Subject: [PATCH 212/279] Gzip: compatibility with recent zlib-ng 2.2.x versions. It now uses 5/4 times more memory for the pending buffer. Further, a single allocation is now used, which takes additional 56 bytes for deflate_allocs in 64-bit mode aligned to 16, to store sub-allocation pointers, and the total allocation size now padded up to 128 bytes, which takes theoretically 200 additional bytes in total. This fits though into "4 * (64 + sizeof(void*))" additional space for ZALLOC used in zlib-ng 2.1.x versions. The comment was updated to reflect this. --- src/http/modules/ngx_http_gzip_filter_module.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/http/modules/ngx_http_gzip_filter_module.c b/src/http/modules/ngx_http_gzip_filter_module.c index b555278454c..7113df695aa 100644 --- a/src/http/modules/ngx_http_gzip_filter_module.c +++ b/src/http/modules/ngx_http_gzip_filter_module.c @@ -516,8 +516,10 @@ ngx_http_gzip_filter_memory(ngx_http_request_t *r, ngx_http_gzip_ctx_t *ctx) /* * Another zlib variant, https://github.com/zlib-ng/zlib-ng. * It used to force window bits to 13 for fast compression level, - * uses (64 + sizeof(void*)) additional space on all allocations - * for alignment, 16-byte padding in one of window-sized buffers, + * used (64 + sizeof(void*)) additional space on all allocations + * for alignment and 16-byte padding in one of window-sized buffers, + * uses a single allocation with up to 200 bytes for alignment and + * internal pointers, 5/4 times more memory for the pending buffer, * and 128K hash. */ @@ -526,7 +528,7 @@ ngx_http_gzip_filter_memory(ngx_http_request_t *r, ngx_http_gzip_ctx_t *ctx) } ctx->allocated = 8192 + 16 + (1 << (wbits + 2)) - + 131072 + (1 << (memlevel + 8)) + + 131072 + (5 << (memlevel + 6)) + 4 * (64 + sizeof(void*)); ctx->zlib_ng = 1; } From 47f862ffad6a7068100d50887c495f80973ca47b Mon Sep 17 00:00:00 2001 From: Daniel Vasquez Lopez Date: Thu, 21 Nov 2024 14:27:07 -0800 Subject: [PATCH 213/279] Slice filter: log the expected range in case of range error. --- src/http/modules/ngx_http_slice_filter_module.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/http/modules/ngx_http_slice_filter_module.c b/src/http/modules/ngx_http_slice_filter_module.c index 186380a2f3c..3b0bef629e5 100644 --- a/src/http/modules/ngx_http_slice_filter_module.c +++ b/src/http/modules/ngx_http_slice_filter_module.c @@ -165,8 +165,8 @@ ngx_http_slice_header_filter(ngx_http_request_t *r) if (cr.start != ctx->start || cr.end != end) { ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, - "unexpected range in slice response: %O-%O", - cr.start, cr.end); + "unexpected range in slice response: %O-%O, " + "expected: %O-%O", cr.start, cr.end, ctx->start, end); return NGX_ERROR; } From 8311e14ae614529aabe9e72e87051d191b723fb4 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Wed, 18 Dec 2024 20:03:35 +0400 Subject: [PATCH 214/279] SSL: object cache inheritance from the old configuration cycle. Memory based objects are always inherited, engine based objects are never inherited to adhere the volatile nature of engines, file based objects are inherited subject to modification time and file index. The previous behaviour to bypass cache from the old configuration cycle is preserved with a new directive "ssl_object_cache_inheritable off;". --- src/event/ngx_event_openssl_cache.c | 107 +++++++++++++++++++++++++--- 1 file changed, 99 insertions(+), 8 deletions(-) diff --git a/src/event/ngx_event_openssl_cache.c b/src/event/ngx_event_openssl_cache.c index 8829e28794d..9d196275910 100644 --- a/src/event/ngx_event_openssl_cache.c +++ b/src/event/ngx_event_openssl_cache.c @@ -14,6 +14,14 @@ #define NGX_SSL_CACHE_ENGINE 2 +#define ngx_ssl_cache_get_conf(cycle) \ + (ngx_ssl_cache_t *) ngx_get_conf(cycle->conf_ctx, ngx_openssl_cache_module) + +#define ngx_ssl_cache_get_old_conf(cycle) \ + cycle->old_cycle->conf_ctx ? ngx_ssl_cache_get_conf(cycle->old_cycle) \ + : NULL + + typedef struct { unsigned type:2; unsigned len:30; @@ -39,12 +47,17 @@ typedef struct { ngx_ssl_cache_key_t id; ngx_ssl_cache_type_t *type; void *value; + + time_t mtime; + ngx_file_uniq_t uniq; } ngx_ssl_cache_node_t; typedef struct { ngx_rbtree_t rbtree; ngx_rbtree_node_t sentinel; + + ngx_flag_t inheritable; } ngx_ssl_cache_t; @@ -76,22 +89,36 @@ static void *ngx_ssl_cache_ca_create(ngx_ssl_cache_key_t *id, char **err, static BIO *ngx_ssl_cache_create_bio(ngx_ssl_cache_key_t *id, char **err); static void *ngx_openssl_cache_create_conf(ngx_cycle_t *cycle); +static char *ngx_openssl_cache_init_conf(ngx_cycle_t *cycle, void *conf); static void ngx_ssl_cache_cleanup(void *data); static void ngx_ssl_cache_node_insert(ngx_rbtree_node_t *temp, ngx_rbtree_node_t *node, ngx_rbtree_node_t *sentinel); +static ngx_command_t ngx_openssl_cache_commands[] = { + + { ngx_string("ssl_object_cache_inheritable"), + NGX_MAIN_CONF|NGX_DIRECT_CONF|NGX_CONF_FLAG, + ngx_conf_set_flag_slot, + 0, + offsetof(ngx_ssl_cache_t, inheritable), + NULL }, + + ngx_null_command +}; + + static ngx_core_module_t ngx_openssl_cache_module_ctx = { ngx_string("openssl_cache"), ngx_openssl_cache_create_conf, - NULL + ngx_openssl_cache_init_conf }; ngx_module_t ngx_openssl_cache_module = { NGX_MODULE_V1, &ngx_openssl_cache_module_ctx, /* module context */ - NULL, /* module directives */ + ngx_openssl_cache_commands, /* module directives */ NGX_CORE_MODULE, /* module type */ NULL, /* init master */ NULL, /* init module */ @@ -132,8 +159,13 @@ void * ngx_ssl_cache_fetch(ngx_conf_t *cf, ngx_uint_t index, char **err, ngx_str_t *path, void *data) { + void *value; + time_t mtime; uint32_t hash; - ngx_ssl_cache_t *cache; + ngx_int_t rc; + ngx_file_uniq_t uniq; + ngx_file_info_t fi; + ngx_ssl_cache_t *cache, *old_cache; ngx_ssl_cache_key_t id; ngx_ssl_cache_type_t *type; ngx_ssl_cache_node_t *cn; @@ -151,12 +183,60 @@ ngx_ssl_cache_fetch(ngx_conf_t *cf, ngx_uint_t index, char **err, hash = ngx_murmur_hash2(id.data, id.len); cn = ngx_ssl_cache_lookup(cache, type, &id, hash); + if (cn != NULL) { return type->ref(err, cn->value); } + value = NULL; + + if (id.type == NGX_SSL_CACHE_PATH + && (rc = ngx_file_info(id.data, &fi)) != NGX_FILE_ERROR) + { + mtime = ngx_file_mtime(&fi); + uniq = ngx_file_uniq(&fi); + + } else { + rc = NGX_FILE_ERROR; + mtime = 0; + uniq = 0; + } + + /* try to use a reference from the old cycle */ + + old_cache = ngx_ssl_cache_get_old_conf(cf->cycle); + + if (old_cache && old_cache->inheritable) { + cn = ngx_ssl_cache_lookup(old_cache, type, &id, hash); + + if (cn != NULL) { + switch (id.type) { + + case NGX_SSL_CACHE_DATA: + value = type->ref(err, cn->value); + break; + + default: + if (rc != NGX_FILE_ERROR + && uniq == cn->uniq && mtime == cn->mtime) + { + value = type->ref(err, cn->value); + } + break; + } + } + } + + if (value == NULL) { + value = type->create(&id, err, data); + if (value == NULL) { + return NULL; + } + } + cn = ngx_palloc(cf->pool, sizeof(ngx_ssl_cache_node_t) + id.len + 1); if (cn == NULL) { + type->free(value); return NULL; } @@ -165,14 +245,12 @@ ngx_ssl_cache_fetch(ngx_conf_t *cf, ngx_uint_t index, char **err, cn->id.len = id.len; cn->id.type = id.type; cn->type = type; + cn->value = value; + cn->mtime = mtime; + cn->uniq = uniq; ngx_cpystrn(cn->id.data, id.data, id.len + 1); - cn->value = type->create(&id, err, data); - if (cn->value == NULL) { - return NULL; - } - ngx_rbtree_insert(&cache->rbtree, &cn->node); return type->ref(err, cn->value); @@ -729,6 +807,8 @@ ngx_openssl_cache_create_conf(ngx_cycle_t *cycle) return NULL; } + cache->inheritable = NGX_CONF_UNSET; + cln = ngx_pool_cleanup_add(cycle->pool, 0); if (cln == NULL) { return NULL; @@ -744,6 +824,17 @@ ngx_openssl_cache_create_conf(ngx_cycle_t *cycle) } +static char * +ngx_openssl_cache_init_conf(ngx_cycle_t *cycle, void *conf) +{ + ngx_ssl_cache_t *cache = conf; + + ngx_conf_init_value(cache->inheritable, 1); + + return NGX_CONF_OK; +} + + static void ngx_ssl_cache_cleanup(void *data) { From 7677d5646aeb761b8b9da5af3eb10c008aae3f90 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Wed, 18 Dec 2024 20:09:58 +0400 Subject: [PATCH 215/279] SSL: encrypted certificate keys are exempt from object cache. SSL object cache, as previously introduced in 1.27.2, did not take into account encrypted certificate keys that might be unexpectedly fetched from the cache regardless of the matching passphrase. To avoid this, caching of encrypted certificate keys is now disabled based on the passphrase callback invocation. A notable exception is encrypted certificate keys configured without ssl_password_file. They are loaded once resulting in the passphrase prompt on startup and reused in other contexts as applicable. --- src/event/ngx_event_openssl_cache.c | 53 +++++++++++++++++++++-------- 1 file changed, 38 insertions(+), 15 deletions(-) diff --git a/src/event/ngx_event_openssl_cache.c b/src/event/ngx_event_openssl_cache.c index 9d196275910..c79f77456ab 100644 --- a/src/event/ngx_event_openssl_cache.c +++ b/src/event/ngx_event_openssl_cache.c @@ -13,6 +13,8 @@ #define NGX_SSL_CACHE_DATA 1 #define NGX_SSL_CACHE_ENGINE 2 +#define NGX_SSL_CACHE_DISABLED (ngx_array_t *) (uintptr_t) -1 + #define ngx_ssl_cache_get_conf(cycle) \ (ngx_ssl_cache_t *) ngx_get_conf(cycle->conf_ctx, ngx_openssl_cache_module) @@ -61,6 +63,12 @@ typedef struct { } ngx_ssl_cache_t; +typedef struct { + ngx_str_t *pwd; + unsigned encrypted:1; +} ngx_ssl_cache_pwd_t; + + static ngx_int_t ngx_ssl_cache_init_key(ngx_pool_t *pool, ngx_uint_t index, ngx_str_t *path, ngx_ssl_cache_key_t *id); static ngx_ssl_cache_node_t *ngx_ssl_cache_lookup(ngx_ssl_cache_t *cache, @@ -228,9 +236,10 @@ ngx_ssl_cache_fetch(ngx_conf_t *cf, ngx_uint_t index, char **err, } if (value == NULL) { - value = type->create(&id, err, data); - if (value == NULL) { - return NULL; + value = type->create(&id, err, &data); + + if (value == NULL || data == NGX_SSL_CACHE_DISABLED) { + return value; } } @@ -269,7 +278,7 @@ ngx_ssl_cache_connection_fetch(ngx_pool_t *pool, ngx_uint_t index, char **err, return NULL; } - return ngx_ssl_cache_types[index].create(&id, err, data); + return ngx_ssl_cache_types[index].create(&id, err, &data); } @@ -472,13 +481,13 @@ ngx_ssl_cache_cert_ref(char **err, void *data) static void * ngx_ssl_cache_pkey_create(ngx_ssl_cache_key_t *id, char **err, void *data) { - ngx_array_t *passwords = data; + ngx_array_t **passwords = data; - BIO *bio; - EVP_PKEY *pkey; - ngx_str_t *pwd; - ngx_uint_t tries; - pem_password_cb *cb; + BIO *bio; + EVP_PKEY *pkey; + ngx_uint_t tries; + pem_password_cb *cb; + ngx_ssl_cache_pwd_t cb_data, *pwd; if (id->type == NGX_SSL_CACHE_ENGINE) { @@ -531,12 +540,16 @@ ngx_ssl_cache_pkey_create(ngx_ssl_cache_key_t *id, char **err, void *data) return NULL; } - if (passwords) { - tries = passwords->nelts; - pwd = passwords->elts; + cb_data.encrypted = 0; + + if (*passwords) { + cb_data.pwd = (*passwords)->elts; + tries = (*passwords)->nelts; + pwd = &cb_data; cb = ngx_ssl_cache_pkey_password_callback; } else { + cb_data.pwd = NULL; tries = 1; pwd = NULL; cb = NULL; @@ -552,7 +565,7 @@ ngx_ssl_cache_pkey_create(ngx_ssl_cache_key_t *id, char **err, void *data) if (tries-- > 1) { ERR_clear_error(); (void) BIO_reset(bio); - pwd++; + cb_data.pwd++; continue; } @@ -561,6 +574,10 @@ ngx_ssl_cache_pkey_create(ngx_ssl_cache_key_t *id, char **err, void *data) return NULL; } + if (cb_data.encrypted) { + *passwords = NGX_SSL_CACHE_DISABLED; + } + BIO_free(bio); return pkey; @@ -571,7 +588,9 @@ static int ngx_ssl_cache_pkey_password_callback(char *buf, int size, int rwflag, void *userdata) { - ngx_str_t *pwd = userdata; + ngx_ssl_cache_pwd_t *data = userdata; + + ngx_str_t *pwd; if (rwflag) { ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, 0, @@ -580,6 +599,10 @@ ngx_ssl_cache_pkey_password_callback(char *buf, int size, int rwflag, return 0; } + data->encrypted = 1; + + pwd = data->pwd; + if (pwd == NULL) { return 0; } From 0e756d67aa1e42e3b1b360936eb4d6c06bced2c1 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Tue, 29 Oct 2024 16:25:11 +0400 Subject: [PATCH 216/279] SSL: caching certificates and certificate keys with variables. A new directive "ssl_certificate_cache max=N [valid=time] [inactive=time]" enables caching of SSL certificate chain and secret key objects specified by "ssl_certificate" and "ssl_certificate_key" directives with variables. Co-authored-by: Aleksei Bavshin --- src/event/ngx_event_openssl.c | 11 +- src/event/ngx_event_openssl.h | 12 +- src/event/ngx_event_openssl_cache.c | 240 ++++++++++++++++++++++--- src/http/modules/ngx_http_ssl_module.c | 106 +++++++++++ src/http/modules/ngx_http_ssl_module.h | 2 + src/http/ngx_http_request.c | 1 + src/http/ngx_http_upstream.c | 2 +- src/stream/ngx_stream_proxy_module.c | 2 +- src/stream/ngx_stream_ssl_module.c | 107 +++++++++++ src/stream/ngx_stream_ssl_module.h | 66 +++---- 10 files changed, 481 insertions(+), 68 deletions(-) diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c index 35e9f3c887d..8963c81244d 100644 --- a/src/event/ngx_event_openssl.c +++ b/src/event/ngx_event_openssl.c @@ -562,15 +562,16 @@ ngx_ssl_certificate(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *cert, ngx_int_t ngx_ssl_connection_certificate(ngx_connection_t *c, ngx_pool_t *pool, - ngx_str_t *cert, ngx_str_t *key, ngx_array_t *passwords) + ngx_str_t *cert, ngx_str_t *key, ngx_ssl_cache_t *cache, + ngx_array_t *passwords) { char *err; X509 *x509; EVP_PKEY *pkey; STACK_OF(X509) *chain; - chain = ngx_ssl_cache_connection_fetch(pool, NGX_SSL_CACHE_CERT, &err, - cert, NULL); + chain = ngx_ssl_cache_connection_fetch(cache, pool, NGX_SSL_CACHE_CERT, + &err, cert, NULL); if (chain == NULL) { if (err != NULL) { ngx_ssl_error(NGX_LOG_ERR, c->log, 0, @@ -610,8 +611,8 @@ ngx_ssl_connection_certificate(ngx_connection_t *c, ngx_pool_t *pool, #endif - pkey = ngx_ssl_cache_connection_fetch(pool, NGX_SSL_CACHE_PKEY, &err, - key, passwords); + pkey = ngx_ssl_cache_connection_fetch(cache, pool, NGX_SSL_CACHE_PKEY, + &err, key, passwords); if (pkey == NULL) { if (err != NULL) { ngx_ssl_error(NGX_LOG_ERR, c->log, 0, diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h index 2147205d610..0713c56713b 100644 --- a/src/event/ngx_event_openssl.h +++ b/src/event/ngx_event_openssl.h @@ -83,7 +83,8 @@ #endif -typedef struct ngx_ssl_ocsp_s ngx_ssl_ocsp_t; +typedef struct ngx_ssl_cache_s ngx_ssl_cache_t; +typedef struct ngx_ssl_ocsp_s ngx_ssl_ocsp_t; struct ngx_ssl_s { @@ -214,7 +215,8 @@ ngx_int_t ngx_ssl_certificates(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_int_t ngx_ssl_certificate(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *cert, ngx_str_t *key, ngx_array_t *passwords); ngx_int_t ngx_ssl_connection_certificate(ngx_connection_t *c, ngx_pool_t *pool, - ngx_str_t *cert, ngx_str_t *key, ngx_array_t *passwords); + ngx_str_t *cert, ngx_str_t *key, ngx_ssl_cache_t *cache, + ngx_array_t *passwords); ngx_int_t ngx_ssl_ciphers(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *ciphers, ngx_uint_t prefer_server_ciphers); @@ -237,10 +239,12 @@ ngx_int_t ngx_ssl_ocsp_get_status(ngx_connection_t *c, const char **s); void ngx_ssl_ocsp_cleanup(ngx_connection_t *c); ngx_int_t ngx_ssl_ocsp_cache_init(ngx_shm_zone_t *shm_zone, void *data); +ngx_ssl_cache_t *ngx_ssl_cache_init(ngx_pool_t *pool, ngx_uint_t max, + time_t valid, time_t inactive); void *ngx_ssl_cache_fetch(ngx_conf_t *cf, ngx_uint_t index, char **err, ngx_str_t *path, void *data); -void *ngx_ssl_cache_connection_fetch(ngx_pool_t *pool, ngx_uint_t index, - char **err, ngx_str_t *path, void *data); +void *ngx_ssl_cache_connection_fetch(ngx_ssl_cache_t *cache, ngx_pool_t *pool, + ngx_uint_t index, char **err, ngx_str_t *path, void *data); ngx_array_t *ngx_ssl_read_password_file(ngx_conf_t *cf, ngx_str_t *file); ngx_array_t *ngx_ssl_preserve_passwords(ngx_conf_t *cf, diff --git a/src/event/ngx_event_openssl_cache.c b/src/event/ngx_event_openssl_cache.c index c79f77456ab..7589e6c90e6 100644 --- a/src/event/ngx_event_openssl_cache.c +++ b/src/event/ngx_event_openssl_cache.c @@ -46,21 +46,31 @@ typedef struct { typedef struct { ngx_rbtree_node_t node; + ngx_queue_t queue; ngx_ssl_cache_key_t id; ngx_ssl_cache_type_t *type; void *value; + time_t created; + time_t accessed; + time_t mtime; ngx_file_uniq_t uniq; } ngx_ssl_cache_node_t; -typedef struct { +struct ngx_ssl_cache_s { ngx_rbtree_t rbtree; ngx_rbtree_node_t sentinel; + ngx_queue_t expire_queue; ngx_flag_t inheritable; -} ngx_ssl_cache_t; + + ngx_uint_t current; + ngx_uint_t max; + time_t valid; + time_t inactive; +}; typedef struct { @@ -73,6 +83,8 @@ static ngx_int_t ngx_ssl_cache_init_key(ngx_pool_t *pool, ngx_uint_t index, ngx_str_t *path, ngx_ssl_cache_key_t *id); static ngx_ssl_cache_node_t *ngx_ssl_cache_lookup(ngx_ssl_cache_t *cache, ngx_ssl_cache_type_t *type, ngx_ssl_cache_key_t *id, uint32_t hash); +static void ngx_ssl_cache_expire(ngx_ssl_cache_t *cache, ngx_uint_t n, + ngx_log_t *log); static void *ngx_ssl_cache_cert_create(ngx_ssl_cache_key_t *id, char **err, void *data); @@ -101,6 +113,8 @@ static char *ngx_openssl_cache_init_conf(ngx_cycle_t *cycle, void *conf); static void ngx_ssl_cache_cleanup(void *data); static void ngx_ssl_cache_node_insert(ngx_rbtree_node_t *temp, ngx_rbtree_node_t *node, ngx_rbtree_node_t *sentinel); +static void ngx_ssl_cache_node_free(ngx_rbtree_t *rbtree, + ngx_ssl_cache_node_t *cn); static ngx_command_t ngx_openssl_cache_commands[] = { @@ -260,6 +274,8 @@ ngx_ssl_cache_fetch(ngx_conf_t *cf, ngx_uint_t index, char **err, ngx_cpystrn(cn->id.data, id.data, id.len + 1); + ngx_queue_init(&cn->queue); + ngx_rbtree_insert(&cache->rbtree, &cn->node); return type->ref(err, cn->value); @@ -267,10 +283,15 @@ ngx_ssl_cache_fetch(ngx_conf_t *cf, ngx_uint_t index, char **err, void * -ngx_ssl_cache_connection_fetch(ngx_pool_t *pool, ngx_uint_t index, char **err, - ngx_str_t *path, void *data) +ngx_ssl_cache_connection_fetch(ngx_ssl_cache_t *cache, ngx_pool_t *pool, + ngx_uint_t index, char **err, ngx_str_t *path, void *data) { - ngx_ssl_cache_key_t id; + void *value; + time_t now; + uint32_t hash; + ngx_ssl_cache_key_t id; + ngx_ssl_cache_type_t *type; + ngx_ssl_cache_node_t *cn; *err = NULL; @@ -278,7 +299,89 @@ ngx_ssl_cache_connection_fetch(ngx_pool_t *pool, ngx_uint_t index, char **err, return NULL; } - return ngx_ssl_cache_types[index].create(&id, err, &data); + type = &ngx_ssl_cache_types[index]; + + if (cache == NULL) { + return type->create(&id, err, &data); + } + + now = ngx_time(); + + hash = ngx_murmur_hash2(id.data, id.len); + + cn = ngx_ssl_cache_lookup(cache, type, &id, hash); + + if (cn != NULL) { + ngx_queue_remove(&cn->queue); + + if (id.type == NGX_SSL_CACHE_DATA) { + goto found; + } + + if (now - cn->created > cache->valid) { + ngx_log_debug1(NGX_LOG_DEBUG_CORE, pool->log, 0, + "update cached ssl object: %s", cn->id.data); + + type->free(cn->value); + + value = type->create(&id, err, &data); + + if (value == NULL || data == NGX_SSL_CACHE_DISABLED) { + ngx_rbtree_delete(&cache->rbtree, &cn->node); + + cache->current--; + + ngx_free(cn); + + return value; + } + + cn->value = value; + cn->created = now; + } + + goto found; + } + + value = type->create(&id, err, &data); + + if (value == NULL || data == NGX_SSL_CACHE_DISABLED) { + return value; + } + + cn = ngx_alloc(sizeof(ngx_ssl_cache_node_t) + id.len + 1, pool->log); + if (cn == NULL) { + type->free(value); + return NULL; + } + + cn->node.key = hash; + cn->id.data = (u_char *)(cn + 1); + cn->id.len = id.len; + cn->id.type = id.type; + cn->type = type; + cn->value = value; + cn->created = now; + + ngx_cpystrn(cn->id.data, id.data, id.len + 1); + + ngx_ssl_cache_expire(cache, 1, pool->log); + + if (cache->current >= cache->max) { + ngx_ssl_cache_expire(cache, 0, pool->log); + } + + ngx_rbtree_insert(&cache->rbtree, &cn->node); + + cache->current++; + +found: + + cn->accessed = now; + + ngx_queue_insert_head(&cache->expire_queue, &cn->queue); + + return type->ref(err, cn->value); } @@ -365,6 +468,37 @@ ngx_ssl_cache_lookup(ngx_ssl_cache_t *cache, ngx_ssl_cache_type_t *type, } +static void +ngx_ssl_cache_expire(ngx_ssl_cache_t *cache, ngx_uint_t n, + ngx_log_t *log) +{ + time_t now; + ngx_queue_t *q; + ngx_ssl_cache_node_t *cn; + + now = ngx_time(); + + while (n < 3) { + + if (ngx_queue_empty(&cache->expire_queue)) { + return; + } + + q = ngx_queue_last(&cache->expire_queue); + + cn = ngx_queue_data(q, ngx_ssl_cache_node_t, queue); + + if (n++ != 0 && now - cn->accessed <= cache->inactive) { + return; + } + + ngx_ssl_cache_node_free(&cache->rbtree, cn); + + cache->current--; + } +} + + static void * ngx_ssl_cache_cert_create(ngx_ssl_cache_key_t *id, char **err, void *data) { @@ -822,27 +956,15 @@ ngx_ssl_cache_create_bio(ngx_ssl_cache_key_t *id, char **err) static void * ngx_openssl_cache_create_conf(ngx_cycle_t *cycle) { - ngx_ssl_cache_t *cache; - ngx_pool_cleanup_t *cln; + ngx_ssl_cache_t *cache; - cache = ngx_pcalloc(cycle->pool, sizeof(ngx_ssl_cache_t)); + cache = ngx_ssl_cache_init(cycle->pool, 0, 0, 0); if (cache == NULL) { return NULL; } cache->inheritable = NGX_CONF_UNSET; - cln = ngx_pool_cleanup_add(cycle->pool, 0); - if (cln == NULL) { - return NULL; - } - - cln->handler = ngx_ssl_cache_cleanup; - cln->data = cache; - - ngx_rbtree_init(&cache->rbtree, &cache->sentinel, - ngx_ssl_cache_node_insert); - return cache; } @@ -858,6 +980,39 @@ ngx_openssl_cache_init_conf(ngx_cycle_t *cycle, void *conf) } +ngx_ssl_cache_t * +ngx_ssl_cache_init(ngx_pool_t *pool, ngx_uint_t max, time_t valid, + time_t inactive) +{ + ngx_ssl_cache_t *cache; + ngx_pool_cleanup_t *cln; + + cache = ngx_pcalloc(pool, sizeof(ngx_ssl_cache_t)); + if (cache == NULL) { + return NULL; + } + + ngx_rbtree_init(&cache->rbtree, &cache->sentinel, + ngx_ssl_cache_node_insert); + + ngx_queue_init(&cache->expire_queue); + + cache->max = max; + cache->valid = valid; + cache->inactive = inactive; + + cln = ngx_pool_cleanup_add(pool, 0); + if (cln == NULL) { + return NULL; + } + + cln->handler = ngx_ssl_cache_cleanup; + cln->data = cache; + + return cache; +} + + static void ngx_ssl_cache_cleanup(void *data) { @@ -873,12 +1028,47 @@ ngx_ssl_cache_cleanup(void *data) return; } - for (node = ngx_rbtree_min(tree->root, tree->sentinel); - node; - node = ngx_rbtree_next(tree, node)) - { + node = ngx_rbtree_min(tree->root, tree->sentinel); + + while (node != NULL) { cn = ngx_rbtree_data(node, ngx_ssl_cache_node_t, node); - cn->type->free(cn->value); + node = ngx_rbtree_next(tree, node); + + ngx_ssl_cache_node_free(tree, cn); + + if (cache->max) { + cache->current--; + } + } + + if (cache->current) { + ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, 0, + "%ui items still left in ssl cache", + cache->current); + } + + if (!ngx_queue_empty(&cache->expire_queue)) { + ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, 0, + "queue still is not empty in ssl cache"); + + } +} + + +static void +ngx_ssl_cache_node_free(ngx_rbtree_t *rbtree, ngx_ssl_cache_node_t *cn) +{ + cn->type->free(cn->value); + + ngx_rbtree_delete(rbtree, &cn->node); + + if (!ngx_queue_empty(&cn->queue)) { + ngx_queue_remove(&cn->queue); + + ngx_log_debug1(NGX_LOG_DEBUG_CORE, ngx_cycle->log, 0, + "delete cached ssl object: %s", cn->id.data); + + ngx_free(cn); } } diff --git a/src/http/modules/ngx_http_ssl_module.c b/src/http/modules/ngx_http_ssl_module.c index 0e892b04d10..dbfe5c08bf1 100644 --- a/src/http/modules/ngx_http_ssl_module.c +++ b/src/http/modules/ngx_http_ssl_module.c @@ -43,6 +43,8 @@ static char *ngx_http_ssl_merge_srv_conf(ngx_conf_t *cf, static ngx_int_t ngx_http_ssl_compile_certificates(ngx_conf_t *cf, ngx_http_ssl_srv_conf_t *conf); +static char *ngx_http_ssl_certificate_cache(ngx_conf_t *cf, ngx_command_t *cmd, + void *conf); static char *ngx_http_ssl_password_file(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); static char *ngx_http_ssl_session_cache(ngx_conf_t *cf, ngx_command_t *cmd, @@ -108,6 +110,13 @@ static ngx_command_t ngx_http_ssl_commands[] = { offsetof(ngx_http_ssl_srv_conf_t, certificate_keys), NULL }, + { ngx_string("ssl_certificate_cache"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE123, + ngx_http_ssl_certificate_cache, + NGX_HTTP_SRV_CONF_OFFSET, + 0, + NULL }, + { ngx_string("ssl_password_file"), NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1, ngx_http_ssl_password_file, @@ -619,6 +628,7 @@ ngx_http_ssl_create_srv_conf(ngx_conf_t *cf) sscf->verify_depth = NGX_CONF_UNSET_UINT; sscf->certificates = NGX_CONF_UNSET_PTR; sscf->certificate_keys = NGX_CONF_UNSET_PTR; + sscf->certificate_cache = NGX_CONF_UNSET_PTR; sscf->passwords = NGX_CONF_UNSET_PTR; sscf->conf_commands = NGX_CONF_UNSET_PTR; sscf->builtin_session_cache = NGX_CONF_UNSET; @@ -664,6 +674,9 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) ngx_conf_merge_ptr_value(conf->certificate_keys, prev->certificate_keys, NULL); + ngx_conf_merge_ptr_value(conf->certificate_cache, prev->certificate_cache, + NULL); + ngx_conf_merge_ptr_value(conf->passwords, prev->passwords, NULL); ngx_conf_merge_str_value(conf->dhparam, prev->dhparam, ""); @@ -984,6 +997,99 @@ ngx_http_ssl_compile_certificates(ngx_conf_t *cf, } +static char * +ngx_http_ssl_certificate_cache(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) +{ + ngx_http_ssl_srv_conf_t *sscf = conf; + + time_t inactive, valid; + ngx_str_t *value, s; + ngx_int_t max; + ngx_uint_t i; + + if (sscf->certificate_cache != NGX_CONF_UNSET_PTR) { + return "is duplicate"; + } + + value = cf->args->elts; + + max = 0; + inactive = 10; + valid = 60; + + for (i = 1; i < cf->args->nelts; i++) { + + if (ngx_strncmp(value[i].data, "max=", 4) == 0) { + + max = ngx_atoi(value[i].data + 4, value[i].len - 4); + if (max <= 0) { + goto failed; + } + + continue; + } + + if (ngx_strncmp(value[i].data, "inactive=", 9) == 0) { + + s.len = value[i].len - 9; + s.data = value[i].data + 9; + + inactive = ngx_parse_time(&s, 1); + if (inactive == (time_t) NGX_ERROR) { + goto failed; + } + + continue; + } + + if (ngx_strncmp(value[i].data, "valid=", 6) == 0) { + + s.len = value[i].len - 6; + s.data = value[i].data + 6; + + valid = ngx_parse_time(&s, 1); + if (valid == (time_t) NGX_ERROR) { + goto failed; + } + + continue; + } + + if (ngx_strcmp(value[i].data, "off") == 0) { + + sscf->certificate_cache = NULL; + + continue; + } + + failed: + + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "invalid parameter \"%V\"", &value[i]); + return NGX_CONF_ERROR; + } + + if (sscf->certificate_cache == NULL) { + return NGX_CONF_OK; + } + + if (max == 0) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "\"ssl_certificate_cache\" must have " + "the \"max\" parameter"); + return NGX_CONF_ERROR; + } + + sscf->certificate_cache = ngx_ssl_cache_init(cf->pool, max, valid, + inactive); + if (sscf->certificate_cache == NULL) { + return NGX_CONF_ERROR; + } + + return NGX_CONF_OK; +} + + static char * ngx_http_ssl_password_file(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) { diff --git a/src/http/modules/ngx_http_ssl_module.h b/src/http/modules/ngx_http_ssl_module.h index c69c8ffd2bf..8650fab9376 100644 --- a/src/http/modules/ngx_http_ssl_module.h +++ b/src/http/modules/ngx_http_ssl_module.h @@ -38,6 +38,8 @@ typedef struct { ngx_array_t *certificate_values; ngx_array_t *certificate_key_values; + ngx_ssl_cache_t *certificate_cache; + ngx_str_t dhparam; ngx_str_t ecdh_curve; ngx_str_t client_certificate; diff --git a/src/http/ngx_http_request.c b/src/http/ngx_http_request.c index 3cca57cf5ee..f44c9e79b71 100644 --- a/src/http/ngx_http_request.c +++ b/src/http/ngx_http_request.c @@ -1054,6 +1054,7 @@ ngx_http_ssl_certificate(ngx_ssl_conn_t *ssl_conn, void *arg) "ssl key: \"%s\"", key.data); if (ngx_ssl_connection_certificate(c, r->pool, &cert, &key, + sscf->certificate_cache, sscf->passwords) != NGX_OK) { diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c index d95662c5661..e6382ef7913 100644 --- a/src/http/ngx_http_upstream.c +++ b/src/http/ngx_http_upstream.c @@ -2018,7 +2018,7 @@ ngx_http_upstream_ssl_certificate(ngx_http_request_t *r, ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0, "http upstream ssl key: \"%s\"", key.data); - if (ngx_ssl_connection_certificate(c, r->pool, &cert, &key, + if (ngx_ssl_connection_certificate(c, r->pool, &cert, &key, NULL, u->conf->ssl_passwords) != NGX_OK) { diff --git a/src/stream/ngx_stream_proxy_module.c b/src/stream/ngx_stream_proxy_module.c index e978056ef0f..21b579af325 100644 --- a/src/stream/ngx_stream_proxy_module.c +++ b/src/stream/ngx_stream_proxy_module.c @@ -1324,7 +1324,7 @@ ngx_stream_proxy_ssl_certificate(ngx_stream_session_t *s) ngx_log_debug1(NGX_LOG_DEBUG_STREAM, c->log, 0, "stream upstream ssl key: \"%s\"", key.data); - if (ngx_ssl_connection_certificate(c, c->pool, &cert, &key, + if (ngx_ssl_connection_certificate(c, c->pool, &cert, &key, NULL, pscf->ssl_passwords) != NGX_OK) { diff --git a/src/stream/ngx_stream_ssl_module.c b/src/stream/ngx_stream_ssl_module.c index dfbaa0e2f6a..b84995d61f4 100644 --- a/src/stream/ngx_stream_ssl_module.c +++ b/src/stream/ngx_stream_ssl_module.c @@ -47,6 +47,8 @@ static char *ngx_stream_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, static ngx_int_t ngx_stream_ssl_compile_certificates(ngx_conf_t *cf, ngx_stream_ssl_srv_conf_t *conf); +static char *ngx_stream_ssl_certificate_cache(ngx_conf_t *cf, + ngx_command_t *cmd, void *conf); static char *ngx_stream_ssl_password_file(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); static char *ngx_stream_ssl_session_cache(ngx_conf_t *cf, ngx_command_t *cmd, @@ -117,6 +119,13 @@ static ngx_command_t ngx_stream_ssl_commands[] = { offsetof(ngx_stream_ssl_srv_conf_t, certificate_keys), NULL }, + { ngx_string("ssl_certificate_cache"), + NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_TAKE123, + ngx_stream_ssl_certificate_cache, + NGX_STREAM_SRV_CONF_OFFSET, + 0, + NULL }, + { ngx_string("ssl_password_file"), NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_TAKE1, ngx_stream_ssl_password_file, @@ -718,6 +727,7 @@ ngx_stream_ssl_certificate(ngx_ssl_conn_t *ssl_conn, void *arg) "ssl key: \"%s\"", key.data); if (ngx_ssl_connection_certificate(c, c->pool, &cert, &key, + sscf->certificate_cache, sscf->passwords) != NGX_OK) { @@ -844,6 +854,7 @@ ngx_stream_ssl_create_srv_conf(ngx_conf_t *cf) sscf->handshake_timeout = NGX_CONF_UNSET_MSEC; sscf->certificates = NGX_CONF_UNSET_PTR; sscf->certificate_keys = NGX_CONF_UNSET_PTR; + sscf->certificate_cache = NGX_CONF_UNSET_PTR; sscf->passwords = NGX_CONF_UNSET_PTR; sscf->conf_commands = NGX_CONF_UNSET_PTR; sscf->prefer_server_ciphers = NGX_CONF_UNSET; @@ -892,6 +903,9 @@ ngx_stream_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) ngx_conf_merge_ptr_value(conf->certificate_keys, prev->certificate_keys, NULL); + ngx_conf_merge_ptr_value(conf->certificate_cache, prev->certificate_cache, + NULL); + ngx_conf_merge_ptr_value(conf->passwords, prev->passwords, NULL); ngx_conf_merge_str_value(conf->dhparam, prev->dhparam, ""); @@ -1202,6 +1216,99 @@ ngx_stream_ssl_compile_certificates(ngx_conf_t *cf, } +static char * +ngx_stream_ssl_certificate_cache(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) +{ + ngx_stream_ssl_srv_conf_t *sscf = conf; + + time_t inactive, valid; + ngx_str_t *value, s; + ngx_int_t max; + ngx_uint_t i; + + if (sscf->certificate_cache != NGX_CONF_UNSET_PTR) { + return "is duplicate"; + } + + value = cf->args->elts; + + max = 0; + inactive = 10; + valid = 60; + + for (i = 1; i < cf->args->nelts; i++) { + + if (ngx_strncmp(value[i].data, "max=", 4) == 0) { + + max = ngx_atoi(value[i].data + 4, value[i].len - 4); + if (max <= 0) { + goto failed; + } + + continue; + } + + if (ngx_strncmp(value[i].data, "inactive=", 9) == 0) { + + s.len = value[i].len - 9; + s.data = value[i].data + 9; + + inactive = ngx_parse_time(&s, 1); + if (inactive == (time_t) NGX_ERROR) { + goto failed; + } + + continue; + } + + if (ngx_strncmp(value[i].data, "valid=", 6) == 0) { + + s.len = value[i].len - 6; + s.data = value[i].data + 6; + + valid = ngx_parse_time(&s, 1); + if (valid == (time_t) NGX_ERROR) { + goto failed; + } + + continue; + } + + if (ngx_strcmp(value[i].data, "off") == 0) { + + sscf->certificate_cache = NULL; + + continue; + } + + failed: + + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "invalid parameter \"%V\"", &value[i]); + return NGX_CONF_ERROR; + } + + if (sscf->certificate_cache == NULL) { + return NGX_CONF_OK; + } + + if (max == 0) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "\"ssl_certificate_cache\" must have " + "the \"max\" parameter"); + return NGX_CONF_ERROR; + } + + sscf->certificate_cache = ngx_ssl_cache_init(cf->pool, max, valid, + inactive); + if (sscf->certificate_cache == NULL) { + return NGX_CONF_ERROR; + } + + return NGX_CONF_OK; +} + + static char * ngx_stream_ssl_password_file(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) { diff --git a/src/stream/ngx_stream_ssl_module.h b/src/stream/ngx_stream_ssl_module.h index e6769426c8e..ffa03a6f343 100644 --- a/src/stream/ngx_stream_ssl_module.h +++ b/src/stream/ngx_stream_ssl_module.h @@ -15,53 +15,55 @@ typedef struct { - ngx_msec_t handshake_timeout; + ngx_msec_t handshake_timeout; - ngx_flag_t prefer_server_ciphers; - ngx_flag_t reject_handshake; + ngx_flag_t prefer_server_ciphers; + ngx_flag_t reject_handshake; - ngx_ssl_t ssl; + ngx_ssl_t ssl; - ngx_uint_t protocols; + ngx_uint_t protocols; - ngx_uint_t verify; - ngx_uint_t verify_depth; + ngx_uint_t verify; + ngx_uint_t verify_depth; - ssize_t builtin_session_cache; + ssize_t builtin_session_cache; - time_t session_timeout; + time_t session_timeout; - ngx_array_t *certificates; - ngx_array_t *certificate_keys; + ngx_array_t *certificates; + ngx_array_t *certificate_keys; - ngx_array_t *certificate_values; - ngx_array_t *certificate_key_values; + ngx_array_t *certificate_values; + ngx_array_t *certificate_key_values; - ngx_str_t dhparam; - ngx_str_t ecdh_curve; - ngx_str_t client_certificate; - ngx_str_t trusted_certificate; - ngx_str_t crl; - ngx_str_t alpn; + ngx_ssl_cache_t *certificate_cache; - ngx_str_t ciphers; + ngx_str_t dhparam; + ngx_str_t ecdh_curve; + ngx_str_t client_certificate; + ngx_str_t trusted_certificate; + ngx_str_t crl; + ngx_str_t alpn; - ngx_array_t *passwords; - ngx_array_t *conf_commands; + ngx_str_t ciphers; - ngx_shm_zone_t *shm_zone; + ngx_array_t *passwords; + ngx_array_t *conf_commands; - ngx_flag_t session_tickets; - ngx_array_t *session_ticket_keys; + ngx_shm_zone_t *shm_zone; - ngx_uint_t ocsp; - ngx_str_t ocsp_responder; - ngx_shm_zone_t *ocsp_cache_zone; + ngx_flag_t session_tickets; + ngx_array_t *session_ticket_keys; - ngx_flag_t stapling; - ngx_flag_t stapling_verify; - ngx_str_t stapling_file; - ngx_str_t stapling_responder; + ngx_uint_t ocsp; + ngx_str_t ocsp_responder; + ngx_shm_zone_t *ocsp_cache_zone; + + ngx_flag_t stapling; + ngx_flag_t stapling_verify; + ngx_str_t stapling_file; + ngx_str_t stapling_responder; } ngx_stream_ssl_srv_conf_t; From 4b96ad14f3607ab39b160715aeba721097ac4da4 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Mon, 13 Jan 2025 21:40:04 +0400 Subject: [PATCH 217/279] SSL: cache revalidation of file based dynamic certificates. Revalidation is based on file modification time and uniq file index, and happens after the cache object validity time is expired. --- src/event/ngx_event_openssl_cache.c | 44 +++++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/src/event/ngx_event_openssl_cache.c b/src/event/ngx_event_openssl_cache.c index 7589e6c90e6..eb03e16b208 100644 --- a/src/event/ngx_event_openssl_cache.c +++ b/src/event/ngx_event_openssl_cache.c @@ -289,6 +289,7 @@ ngx_ssl_cache_connection_fetch(ngx_ssl_cache_t *cache, ngx_pool_t *pool, void *value; time_t now; uint32_t hash; + ngx_file_info_t fi; ngx_ssl_cache_key_t id; ngx_ssl_cache_type_t *type; ngx_ssl_cache_node_t *cn; @@ -318,7 +319,33 @@ ngx_ssl_cache_connection_fetch(ngx_ssl_cache_t *cache, ngx_pool_t *pool, goto found; } - if (now - cn->created > cache->valid) { + if (now - cn->created <= cache->valid) { + goto found; + } + + switch (id.type) { + + case NGX_SSL_CACHE_PATH: + + if (ngx_file_info(id.data, &fi) != NGX_FILE_ERROR) { + + if (ngx_file_uniq(&fi) == cn->uniq + && ngx_file_mtime(&fi) == cn->mtime) + { + break; + } + + cn->mtime = ngx_file_mtime(&fi); + cn->uniq = ngx_file_uniq(&fi); + + } else { + cn->mtime = 0; + cn->uniq = 0; + } + + /* fall through */ + + default: ngx_log_debug1(NGX_LOG_DEBUG_CORE, pool->log, 0, "update cached ssl object: %s", cn->id.data); @@ -337,9 +364,10 @@ ngx_ssl_cache_connection_fetch(ngx_ssl_cache_t *cache, ngx_pool_t *pool, } cn->value = value; - cn->created = now; } + cn->created = now; + goto found; } @@ -365,6 +393,18 @@ ngx_ssl_cache_connection_fetch(ngx_ssl_cache_t *cache, ngx_pool_t *pool, ngx_cpystrn(cn->id.data, id.data, id.len + 1); + if (id.type == NGX_SSL_CACHE_PATH) { + + if (ngx_file_info(id.data, &fi) != NGX_FILE_ERROR) { + cn->mtime = ngx_file_mtime(&fi); + cn->uniq = ngx_file_uniq(&fi); + + } else { + cn->mtime = 0; + cn->uniq = 0; + } + } + ngx_ssl_cache_expire(cache, 1, pool->log); if (cache->current >= cache->max) { From 454ad0ef33a347eba1a62d18c8fc0498f4dcfd64 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Tue, 29 Oct 2024 18:20:53 +0400 Subject: [PATCH 218/279] Upstream: caching certificates and certificate keys with variables. Caching is enabled with proxy_ssl_certificate_cache and friends. Co-authored-by: Aleksei Bavshin --- src/http/modules/ngx_http_grpc_module.c | 106 ++++++++++++++++++++++ src/http/modules/ngx_http_proxy_module.c | 106 ++++++++++++++++++++++ src/http/modules/ngx_http_uwsgi_module.c | 106 ++++++++++++++++++++++ src/http/ngx_http_upstream.c | 3 +- src/http/ngx_http_upstream.h | 1 + src/stream/ngx_stream_proxy_module.c | 111 ++++++++++++++++++++++- 6 files changed, 431 insertions(+), 2 deletions(-) diff --git a/src/http/modules/ngx_http_grpc_module.c b/src/http/modules/ngx_http_grpc_module.c index 326720447fd..8e246c3cf9b 100644 --- a/src/http/modules/ngx_http_grpc_module.c +++ b/src/http/modules/ngx_http_grpc_module.c @@ -205,6 +205,8 @@ static char *ngx_http_grpc_pass(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); #if (NGX_HTTP_SSL) +static char *ngx_http_grpc_ssl_certificate_cache(ngx_conf_t *cf, + ngx_command_t *cmd, void *conf); static char *ngx_http_grpc_ssl_password_file(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); static char *ngx_http_grpc_ssl_conf_command_check(ngx_conf_t *cf, void *post, @@ -437,6 +439,13 @@ static ngx_command_t ngx_http_grpc_commands[] = { offsetof(ngx_http_grpc_loc_conf_t, upstream.ssl_certificate_key), NULL }, + { ngx_string("grpc_ssl_certificate_cache"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE123, + ngx_http_grpc_ssl_certificate_cache, + NGX_HTTP_LOC_CONF_OFFSET, + 0, + NULL }, + { ngx_string("grpc_ssl_password_file"), NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, ngx_http_grpc_ssl_password_file, @@ -4386,6 +4395,7 @@ ngx_http_grpc_create_loc_conf(ngx_conf_t *cf) conf->ssl_verify_depth = NGX_CONF_UNSET_UINT; conf->upstream.ssl_certificate = NGX_CONF_UNSET_PTR; conf->upstream.ssl_certificate_key = NGX_CONF_UNSET_PTR; + conf->upstream.ssl_certificate_cache = NGX_CONF_UNSET_PTR; conf->upstream.ssl_passwords = NGX_CONF_UNSET_PTR; conf->ssl_conf_commands = NGX_CONF_UNSET_PTR; #endif @@ -4497,6 +4507,8 @@ ngx_http_grpc_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) prev->upstream.ssl_certificate, NULL); ngx_conf_merge_ptr_value(conf->upstream.ssl_certificate_key, prev->upstream.ssl_certificate_key, NULL); + ngx_conf_merge_ptr_value(conf->upstream.ssl_certificate_cache, + prev->upstream.ssl_certificate_cache, NULL); ngx_conf_merge_ptr_value(conf->upstream.ssl_passwords, prev->upstream.ssl_passwords, NULL); @@ -4847,6 +4859,100 @@ ngx_http_grpc_pass(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) #if (NGX_HTTP_SSL) +static char * +ngx_http_grpc_ssl_certificate_cache(ngx_conf_t *cf, ngx_command_t *cmd, + void *conf) +{ + ngx_http_grpc_loc_conf_t *plcf = conf; + + time_t inactive, valid; + ngx_str_t *value, s; + ngx_int_t max; + ngx_uint_t i; + + if (plcf->upstream.ssl_certificate_cache != NGX_CONF_UNSET_PTR) { + return "is duplicate"; + } + + value = cf->args->elts; + + max = 0; + inactive = 10; + valid = 60; + + for (i = 1; i < cf->args->nelts; i++) { + + if (ngx_strncmp(value[i].data, "max=", 4) == 0) { + + max = ngx_atoi(value[i].data + 4, value[i].len - 4); + if (max <= 0) { + goto failed; + } + + continue; + } + + if (ngx_strncmp(value[i].data, "inactive=", 9) == 0) { + + s.len = value[i].len - 9; + s.data = value[i].data + 9; + + inactive = ngx_parse_time(&s, 1); + if (inactive == (time_t) NGX_ERROR) { + goto failed; + } + + continue; + } + + if (ngx_strncmp(value[i].data, "valid=", 6) == 0) { + + s.len = value[i].len - 6; + s.data = value[i].data + 6; + + valid = ngx_parse_time(&s, 1); + if (valid == (time_t) NGX_ERROR) { + goto failed; + } + + continue; + } + + if (ngx_strcmp(value[i].data, "off") == 0) { + + plcf->upstream.ssl_certificate_cache = NULL; + + continue; + } + + failed: + + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "invalid parameter \"%V\"", &value[i]); + return NGX_CONF_ERROR; + } + + if (plcf->upstream.ssl_certificate_cache == NULL) { + return NGX_CONF_OK; + } + + if (max == 0) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "\"grpc_ssl_certificate_cache\" must have " + "the \"max\" parameter"); + return NGX_CONF_ERROR; + } + + plcf->upstream.ssl_certificate_cache = ngx_ssl_cache_init(cf->pool, max, + valid, inactive); + if (plcf->upstream.ssl_certificate_cache == NULL) { + return NGX_CONF_ERROR; + } + + return NGX_CONF_OK; +} + + static char * ngx_http_grpc_ssl_password_file(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) { diff --git a/src/http/modules/ngx_http_proxy_module.c b/src/http/modules/ngx_http_proxy_module.c index 855ef523dd8..27c34fef2a5 100644 --- a/src/http/modules/ngx_http_proxy_module.c +++ b/src/http/modules/ngx_http_proxy_module.c @@ -226,6 +226,8 @@ static char *ngx_http_proxy_cache_key(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); #endif #if (NGX_HTTP_SSL) +static char *ngx_http_proxy_ssl_certificate_cache(ngx_conf_t *cf, + ngx_command_t *cmd, void *conf); static char *ngx_http_proxy_ssl_password_file(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); #endif @@ -775,6 +777,13 @@ static ngx_command_t ngx_http_proxy_commands[] = { offsetof(ngx_http_proxy_loc_conf_t, upstream.ssl_certificate_key), NULL }, + { ngx_string("proxy_ssl_certificate_cache"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE123, + ngx_http_proxy_ssl_certificate_cache, + NGX_HTTP_LOC_CONF_OFFSET, + 0, + NULL }, + { ngx_string("proxy_ssl_password_file"), NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, ngx_http_proxy_ssl_password_file, @@ -3613,6 +3622,7 @@ ngx_http_proxy_create_loc_conf(ngx_conf_t *cf) conf->upstream.ssl_verify = NGX_CONF_UNSET; conf->upstream.ssl_certificate = NGX_CONF_UNSET_PTR; conf->upstream.ssl_certificate_key = NGX_CONF_UNSET_PTR; + conf->upstream.ssl_certificate_cache = NGX_CONF_UNSET_PTR; conf->upstream.ssl_passwords = NGX_CONF_UNSET_PTR; conf->ssl_verify_depth = NGX_CONF_UNSET_UINT; conf->ssl_conf_commands = NGX_CONF_UNSET_PTR; @@ -3964,6 +3974,8 @@ ngx_http_proxy_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) prev->upstream.ssl_certificate, NULL); ngx_conf_merge_ptr_value(conf->upstream.ssl_certificate_key, prev->upstream.ssl_certificate_key, NULL); + ngx_conf_merge_ptr_value(conf->upstream.ssl_certificate_cache, + prev->upstream.ssl_certificate_cache, NULL); ngx_conf_merge_ptr_value(conf->upstream.ssl_passwords, prev->upstream.ssl_passwords, NULL); @@ -5074,6 +5086,100 @@ ngx_http_proxy_cache_key(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) #if (NGX_HTTP_SSL) +static char * +ngx_http_proxy_ssl_certificate_cache(ngx_conf_t *cf, ngx_command_t *cmd, + void *conf) +{ + ngx_http_proxy_loc_conf_t *plcf = conf; + + time_t inactive, valid; + ngx_str_t *value, s; + ngx_int_t max; + ngx_uint_t i; + + if (plcf->upstream.ssl_certificate_cache != NGX_CONF_UNSET_PTR) { + return "is duplicate"; + } + + value = cf->args->elts; + + max = 0; + inactive = 10; + valid = 60; + + for (i = 1; i < cf->args->nelts; i++) { + + if (ngx_strncmp(value[i].data, "max=", 4) == 0) { + + max = ngx_atoi(value[i].data + 4, value[i].len - 4); + if (max <= 0) { + goto failed; + } + + continue; + } + + if (ngx_strncmp(value[i].data, "inactive=", 9) == 0) { + + s.len = value[i].len - 9; + s.data = value[i].data + 9; + + inactive = ngx_parse_time(&s, 1); + if (inactive == (time_t) NGX_ERROR) { + goto failed; + } + + continue; + } + + if (ngx_strncmp(value[i].data, "valid=", 6) == 0) { + + s.len = value[i].len - 6; + s.data = value[i].data + 6; + + valid = ngx_parse_time(&s, 1); + if (valid == (time_t) NGX_ERROR) { + goto failed; + } + + continue; + } + + if (ngx_strcmp(value[i].data, "off") == 0) { + + plcf->upstream.ssl_certificate_cache = NULL; + + continue; + } + + failed: + + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "invalid parameter \"%V\"", &value[i]); + return NGX_CONF_ERROR; + } + + if (plcf->upstream.ssl_certificate_cache == NULL) { + return NGX_CONF_OK; + } + + if (max == 0) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "\"proxy_ssl_certificate_cache\" must have " + "the \"max\" parameter"); + return NGX_CONF_ERROR; + } + + plcf->upstream.ssl_certificate_cache = ngx_ssl_cache_init(cf->pool, max, + valid, inactive); + if (plcf->upstream.ssl_certificate_cache == NULL) { + return NGX_CONF_ERROR; + } + + return NGX_CONF_OK; +} + + static char * ngx_http_proxy_ssl_password_file(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) { diff --git a/src/http/modules/ngx_http_uwsgi_module.c b/src/http/modules/ngx_http_uwsgi_module.c index 7988cc589e2..14aae5bf121 100644 --- a/src/http/modules/ngx_http_uwsgi_module.c +++ b/src/http/modules/ngx_http_uwsgi_module.c @@ -92,6 +92,8 @@ static char *ngx_http_uwsgi_cache_key(ngx_conf_t *cf, ngx_command_t *cmd, #endif #if (NGX_HTTP_SSL) +static char *ngx_http_uwsgi_ssl_certificate_cache(ngx_conf_t *cf, + ngx_command_t *cmd, void *conf); static char *ngx_http_uwsgi_ssl_password_file(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); static char *ngx_http_uwsgi_ssl_conf_command_check(ngx_conf_t *cf, void *post, @@ -559,6 +561,13 @@ static ngx_command_t ngx_http_uwsgi_commands[] = { offsetof(ngx_http_uwsgi_loc_conf_t, upstream.ssl_certificate_key), NULL }, + { ngx_string("uwsgi_ssl_certificate_cache"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE123, + ngx_http_uwsgi_ssl_certificate_cache, + NGX_HTTP_LOC_CONF_OFFSET, + 0, + NULL }, + { ngx_string("uwsgi_ssl_password_file"), NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, ngx_http_uwsgi_ssl_password_file, @@ -1590,6 +1599,7 @@ ngx_http_uwsgi_create_loc_conf(ngx_conf_t *cf) conf->ssl_verify_depth = NGX_CONF_UNSET_UINT; conf->upstream.ssl_certificate = NGX_CONF_UNSET_PTR; conf->upstream.ssl_certificate_key = NGX_CONF_UNSET_PTR; + conf->upstream.ssl_certificate_cache = NGX_CONF_UNSET_PTR; conf->upstream.ssl_passwords = NGX_CONF_UNSET_PTR; conf->ssl_conf_commands = NGX_CONF_UNSET_PTR; #endif @@ -1921,6 +1931,8 @@ ngx_http_uwsgi_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) prev->upstream.ssl_certificate, NULL); ngx_conf_merge_ptr_value(conf->upstream.ssl_certificate_key, prev->upstream.ssl_certificate_key, NULL); + ngx_conf_merge_ptr_value(conf->upstream.ssl_certificate_cache, + prev->upstream.ssl_certificate_cache, NULL); ngx_conf_merge_ptr_value(conf->upstream.ssl_passwords, prev->upstream.ssl_passwords, NULL); @@ -2455,6 +2467,100 @@ ngx_http_uwsgi_cache_key(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) #if (NGX_HTTP_SSL) +static char * +ngx_http_uwsgi_ssl_certificate_cache(ngx_conf_t *cf, ngx_command_t *cmd, + void *conf) +{ + ngx_http_uwsgi_loc_conf_t *plcf = conf; + + time_t inactive, valid; + ngx_str_t *value, s; + ngx_int_t max; + ngx_uint_t i; + + if (plcf->upstream.ssl_certificate_cache != NGX_CONF_UNSET_PTR) { + return "is duplicate"; + } + + value = cf->args->elts; + + max = 0; + inactive = 10; + valid = 60; + + for (i = 1; i < cf->args->nelts; i++) { + + if (ngx_strncmp(value[i].data, "max=", 4) == 0) { + + max = ngx_atoi(value[i].data + 4, value[i].len - 4); + if (max <= 0) { + goto failed; + } + + continue; + } + + if (ngx_strncmp(value[i].data, "inactive=", 9) == 0) { + + s.len = value[i].len - 9; + s.data = value[i].data + 9; + + inactive = ngx_parse_time(&s, 1); + if (inactive == (time_t) NGX_ERROR) { + goto failed; + } + + continue; + } + + if (ngx_strncmp(value[i].data, "valid=", 6) == 0) { + + s.len = value[i].len - 6; + s.data = value[i].data + 6; + + valid = ngx_parse_time(&s, 1); + if (valid == (time_t) NGX_ERROR) { + goto failed; + } + + continue; + } + + if (ngx_strcmp(value[i].data, "off") == 0) { + + plcf->upstream.ssl_certificate_cache = NULL; + + continue; + } + + failed: + + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "invalid parameter \"%V\"", &value[i]); + return NGX_CONF_ERROR; + } + + if (plcf->upstream.ssl_certificate_cache == NULL) { + return NGX_CONF_OK; + } + + if (max == 0) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "\"uwsgi_ssl_certificate_cache\" must have " + "the \"max\" parameter"); + return NGX_CONF_ERROR; + } + + plcf->upstream.ssl_certificate_cache = ngx_ssl_cache_init(cf->pool, max, + valid, inactive); + if (plcf->upstream.ssl_certificate_cache == NULL) { + return NGX_CONF_ERROR; + } + + return NGX_CONF_OK; +} + + static char * ngx_http_uwsgi_ssl_password_file(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) { diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c index e6382ef7913..77dc032f290 100644 --- a/src/http/ngx_http_upstream.c +++ b/src/http/ngx_http_upstream.c @@ -2018,7 +2018,8 @@ ngx_http_upstream_ssl_certificate(ngx_http_request_t *r, ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0, "http upstream ssl key: \"%s\"", key.data); - if (ngx_ssl_connection_certificate(c, r->pool, &cert, &key, NULL, + if (ngx_ssl_connection_certificate(c, r->pool, &cert, &key, + u->conf->ssl_certificate_cache, u->conf->ssl_passwords) != NGX_OK) { diff --git a/src/http/ngx_http_upstream.h b/src/http/ngx_http_upstream.h index 57ee06f8ddd..069c0f7a467 100644 --- a/src/http/ngx_http_upstream.h +++ b/src/http/ngx_http_upstream.h @@ -245,6 +245,7 @@ typedef struct { ngx_http_complex_value_t *ssl_certificate; ngx_http_complex_value_t *ssl_certificate_key; + ngx_ssl_cache_t *ssl_certificate_cache; ngx_array_t *ssl_passwords; #endif diff --git a/src/stream/ngx_stream_proxy_module.c b/src/stream/ngx_stream_proxy_module.c index 21b579af325..7f8bfc4e020 100644 --- a/src/stream/ngx_stream_proxy_module.c +++ b/src/stream/ngx_stream_proxy_module.c @@ -49,6 +49,7 @@ typedef struct { ngx_str_t ssl_crl; ngx_stream_complex_value_t *ssl_certificate; ngx_stream_complex_value_t *ssl_certificate_key; + ngx_ssl_cache_t *ssl_certificate_cache; ngx_array_t *ssl_passwords; ngx_array_t *ssl_conf_commands; @@ -94,6 +95,8 @@ static char *ngx_stream_proxy_bind(ngx_conf_t *cf, ngx_command_t *cmd, #if (NGX_STREAM_SSL) static ngx_int_t ngx_stream_proxy_send_proxy_protocol(ngx_stream_session_t *s); +static char *ngx_stream_proxy_ssl_certificate_cache(ngx_conf_t *cf, + ngx_command_t *cmd, void *conf); static char *ngx_stream_proxy_ssl_password_file(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); static char *ngx_stream_proxy_ssl_conf_command_check(ngx_conf_t *cf, void *post, @@ -341,6 +344,13 @@ static ngx_command_t ngx_stream_proxy_commands[] = { offsetof(ngx_stream_proxy_srv_conf_t, ssl_certificate_key), NULL }, + { ngx_string("proxy_ssl_certificate_cache"), + NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_TAKE123, + ngx_stream_proxy_ssl_certificate_cache, + NGX_STREAM_SRV_CONF_OFFSET, + 0, + NULL }, + { ngx_string("proxy_ssl_password_file"), NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_TAKE1, ngx_stream_proxy_ssl_password_file, @@ -1029,6 +1039,100 @@ ngx_stream_proxy_send_proxy_protocol(ngx_stream_session_t *s) } +static char * +ngx_stream_proxy_ssl_certificate_cache(ngx_conf_t *cf, ngx_command_t *cmd, + void *conf) +{ + ngx_stream_proxy_srv_conf_t *pscf = conf; + + time_t inactive, valid; + ngx_str_t *value, s; + ngx_int_t max; + ngx_uint_t i; + + if (pscf->ssl_certificate_cache != NGX_CONF_UNSET_PTR) { + return "is duplicate"; + } + + value = cf->args->elts; + + max = 0; + inactive = 10; + valid = 60; + + for (i = 1; i < cf->args->nelts; i++) { + + if (ngx_strncmp(value[i].data, "max=", 4) == 0) { + + max = ngx_atoi(value[i].data + 4, value[i].len - 4); + if (max <= 0) { + goto failed; + } + + continue; + } + + if (ngx_strncmp(value[i].data, "inactive=", 9) == 0) { + + s.len = value[i].len - 9; + s.data = value[i].data + 9; + + inactive = ngx_parse_time(&s, 1); + if (inactive == (time_t) NGX_ERROR) { + goto failed; + } + + continue; + } + + if (ngx_strncmp(value[i].data, "valid=", 6) == 0) { + + s.len = value[i].len - 6; + s.data = value[i].data + 6; + + valid = ngx_parse_time(&s, 1); + if (valid == (time_t) NGX_ERROR) { + goto failed; + } + + continue; + } + + if (ngx_strcmp(value[i].data, "off") == 0) { + + pscf->ssl_certificate_cache = NULL; + + continue; + } + + failed: + + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "invalid parameter \"%V\"", &value[i]); + return NGX_CONF_ERROR; + } + + if (pscf->ssl_certificate_cache == NULL) { + return NGX_CONF_OK; + } + + if (max == 0) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "\"proxy_ssl_certificate_cache\" must have " + "the \"max\" parameter"); + return NGX_CONF_ERROR; + } + + pscf->ssl_certificate_cache = ngx_ssl_cache_init(cf->pool, max, valid, + inactive); + if (pscf->ssl_certificate_cache == NULL) { + return NGX_CONF_ERROR; + } + + return NGX_CONF_OK; +} + + static char * ngx_stream_proxy_ssl_password_file(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) @@ -1324,7 +1428,8 @@ ngx_stream_proxy_ssl_certificate(ngx_stream_session_t *s) ngx_log_debug1(NGX_LOG_DEBUG_STREAM, c->log, 0, "stream upstream ssl key: \"%s\"", key.data); - if (ngx_ssl_connection_certificate(c, c->pool, &cert, &key, NULL, + if (ngx_ssl_connection_certificate(c, c->pool, &cert, &key, + pscf->ssl_certificate_cache, pscf->ssl_passwords) != NGX_OK) { @@ -2120,6 +2225,7 @@ ngx_stream_proxy_create_srv_conf(ngx_conf_t *cf) conf->ssl_verify_depth = NGX_CONF_UNSET_UINT; conf->ssl_certificate = NGX_CONF_UNSET_PTR; conf->ssl_certificate_key = NGX_CONF_UNSET_PTR; + conf->ssl_certificate_cache = NGX_CONF_UNSET_PTR; conf->ssl_passwords = NGX_CONF_UNSET_PTR; conf->ssl_conf_commands = NGX_CONF_UNSET_PTR; #endif @@ -2206,6 +2312,9 @@ ngx_stream_proxy_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) ngx_conf_merge_ptr_value(conf->ssl_certificate_key, prev->ssl_certificate_key, NULL); + ngx_conf_merge_ptr_value(conf->ssl_certificate_cache, + prev->ssl_certificate_cache, NULL); + ngx_conf_merge_ptr_value(conf->ssl_passwords, prev->ssl_passwords, NULL); ngx_conf_merge_ptr_value(conf->ssl_conf_commands, From 5d5d9adccfeaff7d5926737ee5dfa43937fe5899 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Wed, 8 Jan 2025 17:50:33 +0400 Subject: [PATCH 219/279] SSL: avoid using mismatched certificate/key cached pairs. This can happen with certificates and certificate keys specified with variables due to partial cache update in various scenarios: - cache expiration with only one element of pair evicted - on-disk update with non-cacheable encrypted keys - non-atomic on-disk update The fix is to retry with fresh data on X509_R_KEY_VALUES_MISMATCH. --- src/event/ngx_event_openssl.c | 28 +++++++++++++++++++++++++--- src/event/ngx_event_openssl.h | 2 ++ src/event/ngx_event_openssl_cache.c | 9 +++++++-- 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c index 8963c81244d..0681ca3a265 100644 --- a/src/event/ngx_event_openssl.c +++ b/src/event/ngx_event_openssl.c @@ -567,10 +567,17 @@ ngx_ssl_connection_certificate(ngx_connection_t *c, ngx_pool_t *pool, { char *err; X509 *x509; + u_long n; EVP_PKEY *pkey; + ngx_uint_t mask; STACK_OF(X509) *chain; - chain = ngx_ssl_cache_connection_fetch(cache, pool, NGX_SSL_CACHE_CERT, + mask = 0; + +retry: + + chain = ngx_ssl_cache_connection_fetch(cache, pool, + NGX_SSL_CACHE_CERT | mask, &err, cert, NULL); if (chain == NULL) { if (err != NULL) { @@ -611,7 +618,8 @@ ngx_ssl_connection_certificate(ngx_connection_t *c, ngx_pool_t *pool, #endif - pkey = ngx_ssl_cache_connection_fetch(cache, pool, NGX_SSL_CACHE_PKEY, + pkey = ngx_ssl_cache_connection_fetch(cache, pool, + NGX_SSL_CACHE_PKEY | mask, &err, key, passwords); if (pkey == NULL) { if (err != NULL) { @@ -624,9 +632,23 @@ ngx_ssl_connection_certificate(ngx_connection_t *c, ngx_pool_t *pool, } if (SSL_use_PrivateKey(c->ssl->connection, pkey) == 0) { + EVP_PKEY_free(pkey); + + /* there can be mismatched pairs on uneven cache update */ + + n = ERR_peek_last_error(); + + if (ERR_GET_LIB(n) == ERR_LIB_X509 + && ERR_GET_REASON(n) == X509_R_KEY_VALUES_MISMATCH + && mask == 0) + { + ERR_clear_error(); + mask = NGX_SSL_CACHE_INVALIDATE; + goto retry; + } + ngx_ssl_error(NGX_LOG_ERR, c->log, 0, "SSL_use_PrivateKey(\"%s\") failed", key->data); - EVP_PKEY_free(pkey); return NGX_ERROR; } diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h index 0713c56713b..c9dc50c75da 100644 --- a/src/event/ngx_event_openssl.h +++ b/src/event/ngx_event_openssl.h @@ -206,6 +206,8 @@ typedef struct { #define NGX_SSL_CACHE_CRL 2 #define NGX_SSL_CACHE_CA 3 +#define NGX_SSL_CACHE_INVALIDATE 0x80000000 + ngx_int_t ngx_ssl_init(ngx_log_t *log); ngx_int_t ngx_ssl_create(ngx_ssl_t *ssl, ngx_uint_t protocols, void *data); diff --git a/src/event/ngx_event_openssl_cache.c b/src/event/ngx_event_openssl_cache.c index eb03e16b208..d62b4c4309c 100644 --- a/src/event/ngx_event_openssl_cache.c +++ b/src/event/ngx_event_openssl_cache.c @@ -289,6 +289,7 @@ ngx_ssl_cache_connection_fetch(ngx_ssl_cache_t *cache, ngx_pool_t *pool, void *value; time_t now; uint32_t hash; + ngx_uint_t invalidate; ngx_file_info_t fi; ngx_ssl_cache_key_t id; ngx_ssl_cache_type_t *type; @@ -296,6 +297,9 @@ ngx_ssl_cache_connection_fetch(ngx_ssl_cache_t *cache, ngx_pool_t *pool, *err = NULL; + invalidate = index & NGX_SSL_CACHE_INVALIDATE; + index &= ~NGX_SSL_CACHE_INVALIDATE; + if (ngx_ssl_cache_init_key(pool, index, path, &id) != NGX_OK) { return NULL; } @@ -319,7 +323,7 @@ ngx_ssl_cache_connection_fetch(ngx_ssl_cache_t *cache, ngx_pool_t *pool, goto found; } - if (now - cn->created <= cache->valid) { + if (!invalidate && now - cn->created <= cache->valid) { goto found; } @@ -329,7 +333,8 @@ ngx_ssl_cache_connection_fetch(ngx_ssl_cache_t *cache, ngx_pool_t *pool, if (ngx_file_info(id.data, &fi) != NGX_FILE_ERROR) { - if (ngx_file_uniq(&fi) == cn->uniq + if (!invalidate + && ngx_file_uniq(&fi) == cn->uniq && ngx_file_mtime(&fi) == cn->mtime) { break; From 5ab4f32e9de1d0c8523d3a22fc20a3067e20b68d Mon Sep 17 00:00:00 2001 From: Pavel Pautov Date: Tue, 21 Jan 2025 18:41:16 -0800 Subject: [PATCH 220/279] Upstream: fixed --with-compat build without SSL, broken by 454ad0e. --- src/core/ngx_core.h | 1 + src/event/ngx_event_openssl.h | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/ngx_core.h b/src/core/ngx_core.h index 88db7dc9851..02890b843ac 100644 --- a/src/core/ngx_core.h +++ b/src/core/ngx_core.h @@ -26,6 +26,7 @@ typedef struct ngx_event_aio_s ngx_event_aio_t; typedef struct ngx_connection_s ngx_connection_t; typedef struct ngx_thread_task_s ngx_thread_task_t; typedef struct ngx_ssl_s ngx_ssl_t; +typedef struct ngx_ssl_cache_s ngx_ssl_cache_t; typedef struct ngx_proxy_protocol_s ngx_proxy_protocol_t; typedef struct ngx_quic_stream_s ngx_quic_stream_t; typedef struct ngx_ssl_connection_s ngx_ssl_connection_t; diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h index c9dc50c75da..9ad4d177b14 100644 --- a/src/event/ngx_event_openssl.h +++ b/src/event/ngx_event_openssl.h @@ -83,7 +83,6 @@ #endif -typedef struct ngx_ssl_cache_s ngx_ssl_cache_t; typedef struct ngx_ssl_ocsp_s ngx_ssl_ocsp_t; From 64d0795ac41836b6be8fcceba68f1dbb62b4035a Mon Sep 17 00:00:00 2001 From: Aleksei Bavshin Date: Mon, 27 Jan 2025 10:33:25 -0800 Subject: [PATCH 221/279] QUIC: added missing casts in iov_base assignments. This is consistent with the rest of the code and fixes build on systems with non-standard definition of struct iovec (Solaris, Illumos). --- src/event/quic/ngx_event_quic_output.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/event/quic/ngx_event_quic_output.c b/src/event/quic/ngx_event_quic_output.c index ce6aaab227d..f087e2bfa6c 100644 --- a/src/event/quic/ngx_event_quic_output.c +++ b/src/event/quic/ngx_event_quic_output.c @@ -411,7 +411,7 @@ ngx_quic_send_segments(ngx_connection_t *c, u_char *buf, size_t len, ngx_memzero(msg_control, sizeof(msg_control)); iov.iov_len = len; - iov.iov_base = buf; + iov.iov_base = (void *) buf; msg.msg_iov = &iov; msg.msg_iovlen = 1; @@ -699,7 +699,7 @@ ngx_quic_send(ngx_connection_t *c, u_char *buf, size_t len, ngx_memzero(&msg, sizeof(struct msghdr)); iov.iov_len = len; - iov.iov_base = buf; + iov.iov_base = (void *) buf; msg.msg_iov = &iov; msg.msg_iovlen = 1; From e715202220e2260a8ed125eacf5230d1c1eaeec8 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Fri, 17 Jan 2025 17:55:21 +0400 Subject: [PATCH 222/279] Configure: fixed --with-libatomic=DIR with recent libatomic_ops. The build location of the resulting libatomic_ops.a was changed in v7.4.0 after converting libatomic_ops to use libtool. The fix is to use library from the install path, this allows building with both old and new versions. Initially reported here: https://mailman.nginx.org/pipermail/nginx/2018-April/056054.html --- auto/lib/libatomic/conf | 4 ++-- auto/lib/libatomic/make | 11 ++++++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/auto/lib/libatomic/conf b/auto/lib/libatomic/conf index 8c8cb438b82..dfdc1a62207 100644 --- a/auto/lib/libatomic/conf +++ b/auto/lib/libatomic/conf @@ -7,8 +7,8 @@ if [ $NGX_LIBATOMIC != YES ]; then have=NGX_HAVE_LIBATOMIC . auto/have CORE_INCS="$CORE_INCS $NGX_LIBATOMIC/src" - LINK_DEPS="$LINK_DEPS $NGX_LIBATOMIC/src/libatomic_ops.a" - CORE_LIBS="$CORE_LIBS $NGX_LIBATOMIC/src/libatomic_ops.a" + LINK_DEPS="$LINK_DEPS $NGX_LIBATOMIC/build/lib/libatomic_ops.a" + CORE_LIBS="$CORE_LIBS $NGX_LIBATOMIC/build/lib/libatomic_ops.a" else diff --git a/auto/lib/libatomic/make b/auto/lib/libatomic/make index c90318ea122..530c746a6db 100644 --- a/auto/lib/libatomic/make +++ b/auto/lib/libatomic/make @@ -3,14 +3,19 @@ # Copyright (C) Nginx, Inc. + case $NGX_LIBATOMIC in + /*) ngx_prefix="$NGX_LIBATOMIC/build" ;; + *) ngx_prefix="$PWD/$NGX_LIBATOMIC/build" ;; + esac + cat << END >> $NGX_MAKEFILE -$NGX_LIBATOMIC/src/libatomic_ops.a: $NGX_LIBATOMIC/Makefile - cd $NGX_LIBATOMIC && \$(MAKE) +$NGX_LIBATOMIC/build/lib/libatomic_ops.a: $NGX_LIBATOMIC/Makefile + cd $NGX_LIBATOMIC && \$(MAKE) && \$(MAKE) install $NGX_LIBATOMIC/Makefile: $NGX_MAKEFILE cd $NGX_LIBATOMIC \\ && if [ -f Makefile ]; then \$(MAKE) distclean; fi \\ - && ./configure + && ./configure --prefix=$ngx_prefix END From 04914cfbcbab347e13927b5da4b87e3846038563 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Mon, 9 Dec 2024 13:21:03 +0400 Subject: [PATCH 223/279] Misc: moved documentation in generated ZIP archive. The recently added GitHub files now reside in the docs directory. --- misc/GNUmakefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/misc/GNUmakefile b/misc/GNUmakefile index e18a86a63aa..b7e76b94226 100644 --- a/misc/GNUmakefile +++ b/misc/GNUmakefile @@ -93,6 +93,9 @@ zip: export mv $(TEMP)/$(NGINX)/LICENSE $(TEMP)/$(NGINX)/docs.new mv $(TEMP)/$(NGINX)/README.md $(TEMP)/$(NGINX)/docs.new + mv $(TEMP)/$(NGINX)/CODE_OF_CONDUCT.md $(TEMP)/$(NGINX)/docs.new + mv $(TEMP)/$(NGINX)/CONTRIBUTING.md $(TEMP)/$(NGINX)/docs.new + mv $(TEMP)/$(NGINX)/SECURITY.md $(TEMP)/$(NGINX)/docs.new mv $(TEMP)/$(NGINX)/docs/html $(TEMP)/$(NGINX) rm -r $(TEMP)/$(NGINX)/docs From 22a2a225ba87029f0e7bbc09a80ff7cdad23399d Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Wed, 15 Jan 2025 12:42:39 +0400 Subject: [PATCH 224/279] Added "keepalive_min_timeout" directive. The directive sets a timeout during which a keepalive connection will not be closed by nginx for connection reuse or graceful shutdown. The change allows clients that send multiple requests over the same connection without delay or with a small delay between them, to avoid receiving a TCP RST in response to one of them. This excludes network issues and non-graceful shutdown. As a side-effect, it also addresses the TCP reset problem described in RFC 9112, Section 9.6, when the last sent HTTP response could be damaged by a followup TCP RST. It is important for non-idempotent requests, which cannot be retried by client. It is not recommended to set keepalive_min_timeout to large values as this can introduce an additional delay during graceful shutdown and may restrict nginx from effective connection reuse. --- src/http/ngx_http_core_module.c | 10 +++++++ src/http/ngx_http_core_module.h | 1 + src/http/ngx_http_request.c | 50 ++++++++++++++++++++++++++++----- src/http/ngx_http_request.h | 2 ++ 4 files changed, 56 insertions(+), 7 deletions(-) diff --git a/src/http/ngx_http_core_module.c b/src/http/ngx_http_core_module.c index 033a3bf6403..a1540c01858 100644 --- a/src/http/ngx_http_core_module.c +++ b/src/http/ngx_http_core_module.c @@ -509,6 +509,13 @@ static ngx_command_t ngx_http_core_commands[] = { 0, NULL }, + { ngx_string("keepalive_min_timeout"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, + ngx_conf_set_msec_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_core_loc_conf_t, keepalive_min_timeout), + NULL }, + { ngx_string("keepalive_requests"), NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, ngx_conf_set_num_slot, @@ -3606,6 +3613,7 @@ ngx_http_core_create_loc_conf(ngx_conf_t *cf) clcf->keepalive_time = NGX_CONF_UNSET_MSEC; clcf->keepalive_timeout = NGX_CONF_UNSET_MSEC; clcf->keepalive_header = NGX_CONF_UNSET; + clcf->keepalive_min_timeout = NGX_CONF_UNSET_MSEC; clcf->keepalive_requests = NGX_CONF_UNSET_UINT; clcf->lingering_close = NGX_CONF_UNSET_UINT; clcf->lingering_time = NGX_CONF_UNSET_MSEC; @@ -3844,6 +3852,8 @@ ngx_http_core_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) prev->keepalive_timeout, 75000); ngx_conf_merge_sec_value(conf->keepalive_header, prev->keepalive_header, 0); + ngx_conf_merge_msec_value(conf->keepalive_min_timeout, + prev->keepalive_min_timeout, 0); ngx_conf_merge_uint_value(conf->keepalive_requests, prev->keepalive_requests, 1000); ngx_conf_merge_uint_value(conf->lingering_close, diff --git a/src/http/ngx_http_core_module.h b/src/http/ngx_http_core_module.h index 765e7ff6048..e7e266bf80f 100644 --- a/src/http/ngx_http_core_module.h +++ b/src/http/ngx_http_core_module.h @@ -370,6 +370,7 @@ struct ngx_http_core_loc_conf_s { ngx_msec_t send_timeout; /* send_timeout */ ngx_msec_t keepalive_time; /* keepalive_time */ ngx_msec_t keepalive_timeout; /* keepalive_timeout */ + ngx_msec_t keepalive_min_timeout; /* keepalive_min_timeout */ ngx_msec_t lingering_time; /* lingering_time */ ngx_msec_t lingering_timeout; /* lingering_timeout */ ngx_msec_t resolver_timeout; /* resolver_timeout */ diff --git a/src/http/ngx_http_request.c b/src/http/ngx_http_request.c index f44c9e79b71..0be9da95e8c 100644 --- a/src/http/ngx_http_request.c +++ b/src/http/ngx_http_request.c @@ -2799,6 +2799,13 @@ ngx_http_finalize_connection(ngx_http_request_t *r) r->lingering_close = 1; } + if (r->keepalive + && clcf->keepalive_min_timeout > 0) + { + ngx_http_set_keepalive(r); + return; + } + if (!ngx_terminate && !ngx_exiting && r->keepalive @@ -3301,10 +3308,22 @@ ngx_http_set_keepalive(ngx_http_request_t *r) r->http_state = NGX_HTTP_KEEPALIVE_STATE; #endif - c->idle = 1; - ngx_reusable_connection(c, 1); + if (clcf->keepalive_min_timeout == 0) { + c->idle = 1; + ngx_reusable_connection(c, 1); + } + + if (clcf->keepalive_min_timeout > 0 + && clcf->keepalive_timeout > clcf->keepalive_min_timeout) + { + hc->keepalive_timeout = clcf->keepalive_timeout + - clcf->keepalive_min_timeout; + + } else { + hc->keepalive_timeout = 0; + } - ngx_add_timer(rev, clcf->keepalive_timeout); + ngx_add_timer(rev, clcf->keepalive_timeout - hc->keepalive_timeout); if (rev->ready) { ngx_post_event(rev, &ngx_posted_events); @@ -3315,15 +3334,32 @@ ngx_http_set_keepalive(ngx_http_request_t *r) static void ngx_http_keepalive_handler(ngx_event_t *rev) { - size_t size; - ssize_t n; - ngx_buf_t *b; - ngx_connection_t *c; + size_t size; + ssize_t n; + ngx_buf_t *b; + ngx_connection_t *c; + ngx_http_connection_t *hc; c = rev->data; + hc = c->data; ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "http keepalive handler"); + if (!ngx_terminate + && !ngx_exiting + && rev->timedout + && hc->keepalive_timeout > 0) + { + c->idle = 1; + ngx_reusable_connection(c, 1); + + ngx_add_timer(rev, hc->keepalive_timeout); + + hc->keepalive_timeout = 0; + rev->timedout = 0; + return; + } + if (rev->timedout || c->close) { ngx_http_close_connection(c); return; diff --git a/src/http/ngx_http_request.h b/src/http/ngx_http_request.h index 65c8333f8eb..9407f46ae62 100644 --- a/src/http/ngx_http_request.h +++ b/src/http/ngx_http_request.h @@ -329,6 +329,8 @@ typedef struct { ngx_chain_t *free; + ngx_msec_t keepalive_timeout; + unsigned ssl:1; unsigned proxy_protocol:1; } ngx_http_connection_t; From 46b9f5d389447b3b822ea71f5ac86ebc316c2975 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Wed, 22 Jan 2025 18:55:44 +0400 Subject: [PATCH 225/279] SNI: added restriction for TLSv1.3 cross-SNI session resumption. In OpenSSL, session resumption always happens in the default SSL context, prior to invoking the SNI callback. Further, unlike in TLSv1.2 and older protocols, SSL_get_servername() returns values received in the resumption handshake, which may be different from the value in the initial handshake. Notably, this makes the restriction added in b720f650b insufficient for sessions resumed with different SNI server name. Considering the example from b720f650b, previously, a client was able to request example.org by presenting a certificate for example.org, then to resume and request example.com. The fix is to reject handshakes resumed with a different server name, if verification of client certificates is enabled in a corresponding server configuration. --- src/http/ngx_http_request.c | 27 +++++++++++++++++++++++++-- src/stream/ngx_stream_ssl_module.c | 27 +++++++++++++++++++++++++-- 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/src/http/ngx_http_request.c b/src/http/ngx_http_request.c index 0be9da95e8c..ceac8d307ef 100644 --- a/src/http/ngx_http_request.c +++ b/src/http/ngx_http_request.c @@ -932,6 +932,31 @@ ngx_http_ssl_servername(ngx_ssl_conn_t *ssl_conn, int *ad, void *arg) goto done; } + sscf = ngx_http_get_module_srv_conf(cscf->ctx, ngx_http_ssl_module); + +#if (defined TLS1_3_VERSION \ + && !defined LIBRESSL_VERSION_NUMBER && !defined OPENSSL_IS_BORINGSSL) + + /* + * SSL_SESSION_get0_hostname() is only available in OpenSSL 1.1.1+, + * but servername being negotiated in every TLSv1.3 handshake + * is only returned in OpenSSL 1.1.1+ as well + */ + + if (sscf->verify) { + const char *hostname; + + hostname = SSL_SESSION_get0_hostname(SSL_get0_session(ssl_conn)); + + if (hostname != NULL && ngx_strcmp(hostname, servername) != 0) { + c->ssl->handshake_rejected = 1; + *ad = SSL_AD_ACCESS_DENIED; + return SSL_TLSEXT_ERR_ALERT_FATAL; + } + } + +#endif + hc->ssl_servername = ngx_palloc(c->pool, sizeof(ngx_str_t)); if (hc->ssl_servername == NULL) { goto error; @@ -945,8 +970,6 @@ ngx_http_ssl_servername(ngx_ssl_conn_t *ssl_conn, int *ad, void *arg) ngx_set_connection_log(c, clcf->error_log); - sscf = ngx_http_get_module_srv_conf(hc->conf_ctx, ngx_http_ssl_module); - c->ssl->buffer_size = sscf->buffer_size; if (sscf->ssl.ctx) { diff --git a/src/stream/ngx_stream_ssl_module.c b/src/stream/ngx_stream_ssl_module.c index b84995d61f4..2f1b996246f 100644 --- a/src/stream/ngx_stream_ssl_module.c +++ b/src/stream/ngx_stream_ssl_module.c @@ -589,12 +589,35 @@ ngx_stream_ssl_servername(ngx_ssl_conn_t *ssl_conn, int *ad, void *arg) goto done; } + sscf = ngx_stream_get_module_srv_conf(cscf->ctx, ngx_stream_ssl_module); + +#if (defined TLS1_3_VERSION \ + && !defined LIBRESSL_VERSION_NUMBER && !defined OPENSSL_IS_BORINGSSL) + + /* + * SSL_SESSION_get0_hostname() is only available in OpenSSL 1.1.1+, + * but servername being negotiated in every TLSv1.3 handshake + * is only returned in OpenSSL 1.1.1+ as well + */ + + if (sscf->verify) { + const char *hostname; + + hostname = SSL_SESSION_get0_hostname(SSL_get0_session(ssl_conn)); + + if (hostname != NULL && ngx_strcmp(hostname, servername) != 0) { + c->ssl->handshake_rejected = 1; + *ad = SSL_AD_ACCESS_DENIED; + return SSL_TLSEXT_ERR_ALERT_FATAL; + } + } + +#endif + s->srv_conf = cscf->ctx->srv_conf; ngx_set_connection_log(c, cscf->error_log); - sscf = ngx_stream_get_module_srv_conf(s, ngx_stream_ssl_module); - if (sscf->ssl.ctx) { if (SSL_set_SSL_CTX(ssl_conn, sscf->ssl.ctx) == NULL) { goto error; From ecb809305e54ed15be9f620d56b19ff4e4be7db5 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Wed, 5 Feb 2025 14:24:20 +0400 Subject: [PATCH 226/279] nginx-1.27.4-RELEASE --- docs/xml/nginx/changes.xml | 104 +++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/docs/xml/nginx/changes.xml b/docs/xml/nginx/changes.xml index b469617042c..b55177d80b3 100644 --- a/docs/xml/nginx/changes.xml +++ b/docs/xml/nginx/changes.xml @@ -5,6 +5,110 @@ + + + + +недостаточная проверка в обработке виртуальных серверов +при использовании SNI в TLSv1.3 позволяла повторно использовать +SSL-сессию в контексте другого виртуального сервера, +чтобы обойти проверку клиентских SSL-сертификатов (CVE-2025-23419). + + +insufficient check in virtual servers handling with TLSv1.3 SNI +allowed to reuse SSL sessions in a different virtual server, +to bypass client SSL certificates verification (CVE-2025-23419). + + + + + +директивы ssl_object_cache_inheritable, ssl_certificate_cache, +proxy_ssl_certificate_cache, grpc_ssl_certificate_cache +и uwsgi_ssl_certificate_cache. + + +the "ssl_object_cache_inheritable", "ssl_certificate_cache", +"proxy_ssl_certificate_cache", "grpc_ssl_certificate_cache", +and "uwsgi_ssl_certificate_cache" directives. + + + + + +директива keepalive_min_timeout. + + +the "keepalive_min_timeout" directive. + + + + + +при использовании zlib-ng +в логах появлялись сообщения "gzip filter failed to use preallocated memory". + + +"gzip filter failed to use preallocated memory" alerts appeared in logs +when using zlib-ng. + + + + + +nginx не мог собрать библиотеку libatomic из исходных текстов, +если использовался параметр --with-libatomic=DIR. + + +nginx could not build libatomic library using the library sources +if the --with-libatomic=DIR option was used. + + + + + +могла происходить ошибка установления соединения +при использовании 0-RTT в QUIC; +ошибка появилась в 1.27.1. + + +QUIC connection might not be established when using 0-RTT; +the bug had appeared in 1.27.1. + + + + + +теперь nginx игнорирует пакеты согласования версий QUIC от клиентов. + + +nginx now ignores QUIC version negotiation packets from clients. + + + + + +nginx не собирался на Solaris 10 и более ранних +с модулем ngx_http_v3_module. + + +nginx could not be built on Solaris 10 and earlier +with the ngx_http_v3_module. + + + + + +Исправления в HTTP/3. + + +Bugfixes in HTTP/3. + + + + + + From f274b3f72fa9aa3b2ed9b32817ed4a88eb2256b3 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Tue, 18 Feb 2025 15:38:33 +0400 Subject: [PATCH 227/279] Version bump. --- src/core/nginx.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/nginx.h b/src/core/nginx.h index 29f2c949f28..0907778535d 100644 --- a/src/core/nginx.h +++ b/src/core/nginx.h @@ -9,8 +9,8 @@ #define _NGINX_H_INCLUDED_ -#define nginx_version 1027004 -#define NGINX_VERSION "1.27.4" +#define nginx_version 1027005 +#define NGINX_VERSION "1.27.5" #define NGINX_VER "nginx/" NGINX_VERSION #ifdef NGX_BUILD From 9a4090f02ab438c47178b3b5a4c15a3c769d5027 Mon Sep 17 00:00:00 2001 From: Piotr Sikora Date: Wed, 12 Feb 2025 10:40:58 +0200 Subject: [PATCH 228/279] Core: fix build without libcrypt. libcrypt is no longer part of glibc, so it might not be available. Signed-off-by: Piotr Sikora --- auto/unix | 4 ++-- src/os/unix/ngx_linux_config.h | 6 +++++- src/os/unix/ngx_user.c | 10 +++++++++- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/auto/unix b/auto/unix index f29e69c612d..8bd1b1370dd 100644 --- a/auto/unix +++ b/auto/unix @@ -150,7 +150,7 @@ fi ngx_feature="crypt()" -ngx_feature_name= +ngx_feature_name="NGX_HAVE_CRYPT" ngx_feature_run=no ngx_feature_incs= ngx_feature_path= @@ -162,7 +162,7 @@ ngx_feature_test="crypt(\"test\", \"salt\");" if [ $ngx_found = no ]; then ngx_feature="crypt() in libcrypt" - ngx_feature_name= + ngx_feature_name="NGX_HAVE_CRYPT" ngx_feature_run=no ngx_feature_incs= ngx_feature_path= diff --git a/src/os/unix/ngx_linux_config.h b/src/os/unix/ngx_linux_config.h index 88fef47cefe..d99358c93f3 100644 --- a/src/os/unix/ngx_linux_config.h +++ b/src/os/unix/ngx_linux_config.h @@ -52,7 +52,6 @@ #include /* memalign() */ #include /* IOV_MAX */ #include -#include #include /* uname() */ #include @@ -61,6 +60,11 @@ #include +#if (NGX_HAVE_CRYPT_H) +#include +#endif + + #if (NGX_HAVE_POSIX_SEM) #include #endif diff --git a/src/os/unix/ngx_user.c b/src/os/unix/ngx_user.c index b3d81d07b72..8c769ed9322 100644 --- a/src/os/unix/ngx_user.c +++ b/src/os/unix/ngx_user.c @@ -41,7 +41,7 @@ ngx_libc_crypt(ngx_pool_t *pool, u_char *key, u_char *salt, u_char **encrypted) return NGX_ERROR; } -#else +#elif (NGX_HAVE_CRYPT) ngx_int_t ngx_libc_crypt(ngx_pool_t *pool, u_char *key, u_char *salt, u_char **encrypted) @@ -71,6 +71,14 @@ ngx_libc_crypt(ngx_pool_t *pool, u_char *key, u_char *salt, u_char **encrypted) return NGX_ERROR; } +#else + +ngx_int_t +ngx_libc_crypt(ngx_pool_t *pool, u_char *key, u_char *salt, u_char **encrypted) +{ + return NGX_ERROR; +} + #endif #endif /* NGX_CRYPT */ From 3327353ec05f32bf4ef227fcd67bf40efafa04f8 Mon Sep 17 00:00:00 2001 From: Thierry Bastian Date: Mon, 17 Feb 2025 09:01:27 +0100 Subject: [PATCH 229/279] Configure: MSVC compatibility with PCRE2 10.45. --- auto/lib/pcre/make | 1 + 1 file changed, 1 insertion(+) diff --git a/auto/lib/pcre/make b/auto/lib/pcre/make index 182590ac521..2a987d152da 100644 --- a/auto/lib/pcre/make +++ b/auto/lib/pcre/make @@ -37,6 +37,7 @@ if [ $PCRE_LIBRARY = PCRE2 ]; then pcre2_xclass.c" ngx_pcre_test="pcre2_chkdint.c \ + pcre2_compile_class.c \ pcre2_convert.c \ pcre2_extuni.c \ pcre2_find_bracket.c \ From f51e2de6fe303506538ba939e9071a11387d8275 Mon Sep 17 00:00:00 2001 From: Orgad Shaneh Date: Thu, 13 Feb 2025 22:35:17 +0200 Subject: [PATCH 230/279] Add gitignore file. --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000000..7e5088682b7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +/Makefile +/objs/ +/tmp/ From d25139db01b636a8212c13e1feeca37eaadad0b5 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Tue, 11 Feb 2025 22:54:04 +0400 Subject: [PATCH 231/279] Improved ngx_http_subrequest() error handling. Previously, request might be left in inconsistent state in case of error, which manifested in "http request count is zero" alerts when used by SSI filter. The fix is to reshuffle initialization order to postpone committing state changes until after any potentially failing parts. Found by bad memory allocator simulation. --- src/http/ngx_http_core_module.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/http/ngx_http_core_module.c b/src/http/ngx_http_core_module.c index a1540c01858..92c3eae8a15 100644 --- a/src/http/ngx_http_core_module.c +++ b/src/http/ngx_http_core_module.c @@ -2327,6 +2327,7 @@ ngx_http_subrequest(ngx_http_request_t *r, ngx_connection_t *c; ngx_http_request_t *sr; ngx_http_core_srv_conf_t *cscf; + ngx_http_posted_request_t *posted; ngx_http_postponed_request_t *pr, *p; if (r->subrequests == 0) { @@ -2380,6 +2381,11 @@ ngx_http_subrequest(ngx_http_request_t *r, return NGX_ERROR; } + posted = ngx_palloc(r->pool, sizeof(ngx_http_posted_request_t)); + if (posted == NULL) { + return NGX_ERROR; + } + cscf = ngx_http_get_module_srv_conf(r, ngx_http_core_module); sr->main_conf = cscf->ctx->main_conf; sr->srv_conf = cscf->ctx->srv_conf; @@ -2438,10 +2444,6 @@ ngx_http_subrequest(ngx_http_request_t *r, } if (!sr->background) { - if (c->data == r && r->postponed == NULL) { - c->data = sr; - } - pr = ngx_palloc(r->pool, sizeof(ngx_http_postponed_request_t)); if (pr == NULL) { return NGX_ERROR; @@ -2451,6 +2453,10 @@ ngx_http_subrequest(ngx_http_request_t *r, pr->out = NULL; pr->next = NULL; + if (c->data == r && r->postponed == NULL) { + c->data = sr; + } + if (r->postponed) { for (p = r->postponed; p->next; p = p->next) { /* void */ } p->next = pr; @@ -2498,7 +2504,7 @@ ngx_http_subrequest(ngx_http_request_t *r, ngx_http_update_location_config(sr); } - return ngx_http_post_request(sr, NULL); + return ngx_http_post_request(sr, posted); } From b11ae4cfc9483006f67d92850dc520abe659d880 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Tue, 25 Feb 2025 19:40:22 +0400 Subject: [PATCH 232/279] SSL: style. --- src/event/ngx_event_openssl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c index 0681ca3a265..2446219a7fb 100644 --- a/src/event/ngx_event_openssl.c +++ b/src/event/ngx_event_openssl.c @@ -4034,12 +4034,12 @@ ngx_ssl_get_cached_session(ngx_ssl_conn_t *ssl_conn, const u_char *p; ngx_shm_zone_t *shm_zone; ngx_slab_pool_t *shpool; + ngx_connection_t *c; ngx_rbtree_node_t *node, *sentinel; ngx_ssl_session_t *sess; ngx_ssl_sess_id_t *sess_id; ngx_ssl_session_cache_t *cache; u_char buf[NGX_SSL_MAX_SESSION_SIZE]; - ngx_connection_t *c; hash = ngx_crc32_short((u_char *) (uintptr_t) id, (size_t) len); *copy = 0; From 3d7304b527d1fb6eb697eb8719f286ba7b8e90de Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Fri, 21 Feb 2025 13:49:41 +0400 Subject: [PATCH 233/279] SSL: using static storage for NGX_SSL_MAX_SESSION_SIZE buffers. All such transient buffers are converted to the single storage in BSS. In preparation to raise the limit. --- src/event/ngx_event_openssl.c | 13 +++++++------ src/event/ngx_event_openssl.h | 3 +++ src/http/ngx_http_upstream_round_robin.c | 10 ++++------ src/stream/ngx_stream_upstream_round_robin.c | 10 ++++------ 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c index 2446219a7fb..865c78540a3 100644 --- a/src/event/ngx_event_openssl.c +++ b/src/event/ngx_event_openssl.c @@ -132,6 +132,9 @@ int ngx_ssl_index; int ngx_ssl_certificate_name_index; +u_char ngx_ssl_session_buffer[NGX_SSL_MAX_SESSION_SIZE]; + + ngx_int_t ngx_ssl_init(ngx_log_t *log) { @@ -3889,7 +3892,6 @@ ngx_ssl_new_session(ngx_ssl_conn_t *ssl_conn, ngx_ssl_session_t *sess) ngx_slab_pool_t *shpool; ngx_ssl_sess_id_t *sess_id; ngx_ssl_session_cache_t *cache; - u_char buf[NGX_SSL_MAX_SESSION_SIZE]; #ifdef TLS1_3_VERSION @@ -3916,7 +3918,7 @@ ngx_ssl_new_session(ngx_ssl_conn_t *ssl_conn, ngx_ssl_session_t *sess) return 0; } - p = buf; + p = ngx_ssl_session_buffer; i2d_SSL_SESSION(sess, &p); session_id = (u_char *) SSL_SESSION_get_id(sess, &session_id_length); @@ -3980,7 +3982,7 @@ ngx_ssl_new_session(ngx_ssl_conn_t *ssl_conn, ngx_ssl_session_t *sess) #endif - ngx_memcpy(sess_id->session, buf, len); + ngx_memcpy(sess_id->session, ngx_ssl_session_buffer, len); ngx_memcpy(sess_id->id, session_id, session_id_length); hash = ngx_crc32_short(session_id, session_id_length); @@ -4039,7 +4041,6 @@ ngx_ssl_get_cached_session(ngx_ssl_conn_t *ssl_conn, ngx_ssl_session_t *sess; ngx_ssl_sess_id_t *sess_id; ngx_ssl_session_cache_t *cache; - u_char buf[NGX_SSL_MAX_SESSION_SIZE]; hash = ngx_crc32_short((u_char *) (uintptr_t) id, (size_t) len); *copy = 0; @@ -4087,11 +4088,11 @@ ngx_ssl_get_cached_session(ngx_ssl_conn_t *ssl_conn, if (sess_id->expire > ngx_time()) { slen = sess_id->len; - ngx_memcpy(buf, sess_id->session, slen); + ngx_memcpy(ngx_ssl_session_buffer, sess_id->session, slen); ngx_shmtx_unlock(&shpool->mutex); - p = buf; + p = ngx_ssl_session_buffer; sess = d2i_SSL_SESSION(NULL, &p, slen); return sess; diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h index 9ad4d177b14..25e023b0174 100644 --- a/src/event/ngx_event_openssl.h +++ b/src/event/ngx_event_openssl.h @@ -362,4 +362,7 @@ extern int ngx_ssl_index; extern int ngx_ssl_certificate_name_index; +extern u_char ngx_ssl_session_buffer[NGX_SSL_MAX_SESSION_SIZE]; + + #endif /* _NGX_EVENT_OPENSSL_H_INCLUDED_ */ diff --git a/src/http/ngx_http_upstream_round_robin.c b/src/http/ngx_http_upstream_round_robin.c index 304494b3c37..6b4ff97f206 100644 --- a/src/http/ngx_http_upstream_round_robin.c +++ b/src/http/ngx_http_upstream_round_robin.c @@ -878,7 +878,6 @@ ngx_http_upstream_set_round_robin_peer_session(ngx_peer_connection_t *pc, int len; const u_char *p; ngx_http_upstream_rr_peers_t *peers; - u_char buf[NGX_SSL_MAX_SESSION_SIZE]; #endif peer = rrp->current; @@ -898,12 +897,12 @@ ngx_http_upstream_set_round_robin_peer_session(ngx_peer_connection_t *pc, len = peer->ssl_session_len; - ngx_memcpy(buf, peer->ssl_session, len); + ngx_memcpy(ngx_ssl_session_buffer, peer->ssl_session, len); ngx_http_upstream_rr_peer_unlock(peers, peer); ngx_http_upstream_rr_peers_unlock(peers); - p = buf; + p = ngx_ssl_session_buffer; ssl_session = d2i_SSL_SESSION(NULL, &p, len); rc = ngx_ssl_set_session(pc->connection, ssl_session); @@ -940,7 +939,6 @@ ngx_http_upstream_save_round_robin_peer_session(ngx_peer_connection_t *pc, int len; u_char *p; ngx_http_upstream_rr_peers_t *peers; - u_char buf[NGX_SSL_MAX_SESSION_SIZE]; #endif #if (NGX_HTTP_UPSTREAM_ZONE) @@ -965,7 +963,7 @@ ngx_http_upstream_save_round_robin_peer_session(ngx_peer_connection_t *pc, return; } - p = buf; + p = ngx_ssl_session_buffer; (void) i2d_SSL_SESSION(ssl_session, &p); peer = rrp->current; @@ -995,7 +993,7 @@ ngx_http_upstream_save_round_robin_peer_session(ngx_peer_connection_t *pc, peer->ssl_session_len = len; } - ngx_memcpy(peer->ssl_session, buf, len); + ngx_memcpy(peer->ssl_session, ngx_ssl_session_buffer, len); ngx_http_upstream_rr_peer_unlock(peers, peer); ngx_http_upstream_rr_peers_unlock(peers); diff --git a/src/stream/ngx_stream_upstream_round_robin.c b/src/stream/ngx_stream_upstream_round_robin.c index 5b5f20db771..27db0851e22 100644 --- a/src/stream/ngx_stream_upstream_round_robin.c +++ b/src/stream/ngx_stream_upstream_round_robin.c @@ -911,7 +911,6 @@ ngx_stream_upstream_set_round_robin_peer_session(ngx_peer_connection_t *pc, int len; const u_char *p; ngx_stream_upstream_rr_peers_t *peers; - u_char buf[NGX_SSL_MAX_SESSION_SIZE]; #endif peer = rrp->current; @@ -931,12 +930,12 @@ ngx_stream_upstream_set_round_robin_peer_session(ngx_peer_connection_t *pc, len = peer->ssl_session_len; - ngx_memcpy(buf, peer->ssl_session, len); + ngx_memcpy(ngx_ssl_session_buffer, peer->ssl_session, len); ngx_stream_upstream_rr_peer_unlock(peers, peer); ngx_stream_upstream_rr_peers_unlock(peers); - p = buf; + p = ngx_ssl_session_buffer; ssl_session = d2i_SSL_SESSION(NULL, &p, len); rc = ngx_ssl_set_session(pc->connection, ssl_session); @@ -973,7 +972,6 @@ ngx_stream_upstream_save_round_robin_peer_session(ngx_peer_connection_t *pc, int len; u_char *p; ngx_stream_upstream_rr_peers_t *peers; - u_char buf[NGX_SSL_MAX_SESSION_SIZE]; #endif #if (NGX_STREAM_UPSTREAM_ZONE) @@ -998,7 +996,7 @@ ngx_stream_upstream_save_round_robin_peer_session(ngx_peer_connection_t *pc, return; } - p = buf; + p = ngx_ssl_session_buffer; (void) i2d_SSL_SESSION(ssl_session, &p); peer = rrp->current; @@ -1028,7 +1026,7 @@ ngx_stream_upstream_save_round_robin_peer_session(ngx_peer_connection_t *pc, peer->ssl_session_len = len; } - ngx_memcpy(peer->ssl_session, buf, len); + ngx_memcpy(peer->ssl_session, ngx_ssl_session_buffer, len); ngx_stream_upstream_rr_peer_unlock(peers, peer); ngx_stream_upstream_rr_peers_unlock(peers); From 91245922027767c64e4e6661bf5e7623365c2328 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Tue, 25 Feb 2025 19:50:44 +0400 Subject: [PATCH 234/279] SSL: raised limit for sessions stored in shared memory. Upstream SSL sessions may be of a noticeably larger size with tickets in TLSv1.2 and older versions, or with "stateless" tickets in TLSv1.3, if a client certificate is saved into the session. Further, certain stateless session resumption implemetations may store additional data. Such one is JDK, known to also include server certificates in session ticket data, which roughly doubles a decoded session size to slightly beyond the previous limit. While it's believed to be an issue on the JDK side, this change allows to save such sessions. Another, innocent case is using RSA certificates with 8192 key size. --- src/event/ngx_event_openssl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h index 25e023b0174..b7aaaca7512 100644 --- a/src/event/ngx_event_openssl.h +++ b/src/event/ngx_event_openssl.h @@ -142,7 +142,7 @@ struct ngx_ssl_connection_s { #define NGX_SSL_DFLT_BUILTIN_SCACHE -5 -#define NGX_SSL_MAX_SESSION_SIZE 4096 +#define NGX_SSL_MAX_SESSION_SIZE 8192 typedef struct ngx_ssl_sess_id_s ngx_ssl_sess_id_t; From 311c39037734df89b56325091e9435bc542308f4 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Fri, 21 Feb 2025 15:41:33 +0400 Subject: [PATCH 235/279] SSL: improved logging of saving sessions from upstream servers. This makes it easier to understand why sessions may not be saved in shared memory due to size. --- src/http/ngx_http_upstream_round_robin.c | 6 +++--- src/stream/ngx_stream_upstream_round_robin.c | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/http/ngx_http_upstream_round_robin.c b/src/http/ngx_http_upstream_round_robin.c index 6b4ff97f206..25741869e6a 100644 --- a/src/http/ngx_http_upstream_round_robin.c +++ b/src/http/ngx_http_upstream_round_robin.c @@ -952,11 +952,11 @@ ngx_http_upstream_save_round_robin_peer_session(ngx_peer_connection_t *pc, return; } - ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0, - "save session: %p", ssl_session); - len = i2d_SSL_SESSION(ssl_session, NULL); + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0, + "save session: %p:%d", ssl_session, len); + /* do not cache too big session */ if (len > NGX_SSL_MAX_SESSION_SIZE) { diff --git a/src/stream/ngx_stream_upstream_round_robin.c b/src/stream/ngx_stream_upstream_round_robin.c index 27db0851e22..1dae4e538ac 100644 --- a/src/stream/ngx_stream_upstream_round_robin.c +++ b/src/stream/ngx_stream_upstream_round_robin.c @@ -985,11 +985,11 @@ ngx_stream_upstream_save_round_robin_peer_session(ngx_peer_connection_t *pc, return; } - ngx_log_debug1(NGX_LOG_DEBUG_STREAM, pc->log, 0, - "save session: %p", ssl_session); - len = i2d_SSL_SESSION(ssl_session, NULL); + ngx_log_debug2(NGX_LOG_DEBUG_STREAM, pc->log, 0, + "save session: %p:%d", ssl_session, len); + /* do not cache too big session */ if (len > NGX_SSL_MAX_SESSION_SIZE) { From d16251969bf113272b577920940f020524d5fceb Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Fri, 21 Feb 2025 15:54:04 +0400 Subject: [PATCH 236/279] SSL: removed stale comments. It appears to be a relic from prototype locking removed in b0b7b5a35. --- src/http/ngx_http_upstream_round_robin.c | 2 -- src/stream/ngx_stream_upstream_round_robin.c | 2 -- 2 files changed, 4 deletions(-) diff --git a/src/http/ngx_http_upstream_round_robin.c b/src/http/ngx_http_upstream_round_robin.c index 25741869e6a..4637318d164 100644 --- a/src/http/ngx_http_upstream_round_robin.c +++ b/src/http/ngx_http_upstream_round_robin.c @@ -1021,8 +1021,6 @@ ngx_http_upstream_save_round_robin_peer_session(ngx_peer_connection_t *pc, ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0, "old session: %p", old_ssl_session); - /* TODO: may block */ - ngx_ssl_free_session(old_ssl_session); } } diff --git a/src/stream/ngx_stream_upstream_round_robin.c b/src/stream/ngx_stream_upstream_round_robin.c index 1dae4e538ac..dd80322c8b5 100644 --- a/src/stream/ngx_stream_upstream_round_robin.c +++ b/src/stream/ngx_stream_upstream_round_robin.c @@ -1054,8 +1054,6 @@ ngx_stream_upstream_save_round_robin_peer_session(ngx_peer_connection_t *pc, ngx_log_debug1(NGX_LOG_DEBUG_STREAM, pc->log, 0, "old session: %p", old_ssl_session); - /* TODO: may block */ - ngx_ssl_free_session(old_ssl_session); } } From d31305653701bd99e8e5e6aa48094599a08f9f12 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Thu, 27 Feb 2025 16:09:50 +0400 Subject: [PATCH 237/279] Slice filter: improved memory allocation error handling. As uncovered by recent addition in slice.t, a partially initialized context, coupled with HTTP 206 response from stub backend, might be accessed in the next slice subrequest. Found by bad memory allocator simulation. --- src/http/modules/ngx_http_slice_filter_module.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/http/modules/ngx_http_slice_filter_module.c b/src/http/modules/ngx_http_slice_filter_module.c index 3b0bef629e5..67dc14c8249 100644 --- a/src/http/modules/ngx_http_slice_filter_module.c +++ b/src/http/modules/ngx_http_slice_filter_module.c @@ -419,13 +419,13 @@ ngx_http_slice_range_variable(ngx_http_request_t *r, return NGX_ERROR; } - ngx_http_set_ctx(r, ctx, ngx_http_slice_filter_module); - p = ngx_pnalloc(r->pool, sizeof("bytes=-") - 1 + 2 * NGX_OFF_T_LEN); if (p == NULL) { return NGX_ERROR; } + ngx_http_set_ctx(r, ctx, ngx_http_slice_filter_module); + ctx->start = slcf->size * (ngx_http_slice_get_start(r) / slcf->size); ctx->range.data = p; From a813c639211728a1441945dee149b44a0935f48b Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Thu, 27 Feb 2025 18:42:06 +0400 Subject: [PATCH 238/279] Charset filter: improved validation of charset_map with utf-8. It was possible to write outside of the buffer used to keep UTF-8 decoded values when parsing conversion table configuration. Since this happened before UTF-8 decoding, the fix is to check in advance if character codes are of more than 3-byte sequence. Note that this is already enforced by a later check for ngx_utf8_decode() decoded values for 0xffff, which corresponds to the maximum value encoded as a valid 3-byte sequence, so the fix does not affect the valid values. Found with AddressSanitizer. Fixes GitHub issue #529. --- src/http/modules/ngx_http_charset_filter_module.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/http/modules/ngx_http_charset_filter_module.c b/src/http/modules/ngx_http_charset_filter_module.c index e52b96e9b15..d44da6233a1 100644 --- a/src/http/modules/ngx_http_charset_filter_module.c +++ b/src/http/modules/ngx_http_charset_filter_module.c @@ -1332,6 +1332,12 @@ ngx_http_charset_map(ngx_conf_t *cf, ngx_command_t *dummy, void *conf) table = ctx->table; if (ctx->charset->utf8) { + if (value[1].len / 2 > NGX_UTF_LEN - 1) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "invalid value \"%V\"", &value[1]); + return NGX_CONF_ERROR; + } + p = &table->src2dst[src * NGX_UTF_LEN]; *p++ = (u_char) (value[1].len / 2); From 6c3a9d561271ec451f479a84fbe54c81a63dad2e Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Wed, 5 Feb 2025 19:16:05 +0400 Subject: [PATCH 239/279] Upstream: fixed passwords support for dynamic certificates. Passwords were not preserved in optimized SSL contexts, the bug had appeared in d791b4aab (1.23.1), as in the following configuration: server { proxy_ssl_password_file password; proxy_ssl_certificate $ssl_server_name.crt; proxy_ssl_certificate_key $ssl_server_name.key; location /original/ { proxy_pass https://u1/; } location /optimized/ { proxy_pass https://u2/; } } The fix is to always preserve passwords, by copying to the configuration pool, if dynamic certificates are used. This is done as part of merging "ssl_passwords" configuration. To minimize the number of copies, a preserved version is then used for inheritance. A notable exception is inheritance of preserved empty passwords to the context with statically configured certificates: server { proxy_ssl_certificate $ssl_server_name.crt; proxy_ssl_certificate_key $ssl_server_name.key; location / { proxy_pass ...; proxy_ssl_certificate example.com.crt; proxy_ssl_certificate_key example.com.key; } } In this case, an unmodified version (NULL) of empty passwords is set, to allow reading them from the password prompt on nginx startup. As an additional optimization, a preserved instance of inherited configured passwords is set to the previous level, to inherit it to other contexts: server { proxy_ssl_password_file password; location /1/ { proxy_pass https://u1/; proxy_ssl_certificate $ssl_server_name.crt; proxy_ssl_certificate_key $ssl_server_name.key; } location /2/ { proxy_pass https://u2/; proxy_ssl_certificate $ssl_server_name.crt; proxy_ssl_certificate_key $ssl_server_name.key; } } --- src/http/modules/ngx_http_grpc_module.c | 20 ++++--- src/http/modules/ngx_http_proxy_module.c | 20 ++++--- src/http/modules/ngx_http_uwsgi_module.c | 20 ++++--- src/http/ngx_http_upstream.c | 55 +++++++++++++++++++ src/http/ngx_http_upstream.h | 4 ++ src/stream/ngx_stream_proxy_module.c | 68 ++++++++++++++++++++---- 6 files changed, 144 insertions(+), 43 deletions(-) diff --git a/src/http/modules/ngx_http_grpc_module.c b/src/http/modules/ngx_http_grpc_module.c index 8e246c3cf9b..80046d6a416 100644 --- a/src/http/modules/ngx_http_grpc_module.c +++ b/src/http/modules/ngx_http_grpc_module.c @@ -4509,8 +4509,13 @@ ngx_http_grpc_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) prev->upstream.ssl_certificate_key, NULL); ngx_conf_merge_ptr_value(conf->upstream.ssl_certificate_cache, prev->upstream.ssl_certificate_cache, NULL); - ngx_conf_merge_ptr_value(conf->upstream.ssl_passwords, - prev->upstream.ssl_passwords, NULL); + + if (ngx_http_upstream_merge_ssl_passwords(cf, &conf->upstream, + &prev->upstream) + != NGX_OK) + { + return NGX_CONF_ERROR; + } ngx_conf_merge_ptr_value(conf->ssl_conf_commands, prev->ssl_conf_commands, NULL); @@ -5077,16 +5082,9 @@ ngx_http_grpc_set_ssl(ngx_conf_t *cf, ngx_http_grpc_loc_conf_t *glcf) return NGX_ERROR; } - if (glcf->upstream.ssl_certificate->lengths - || glcf->upstream.ssl_certificate_key->lengths) + if (glcf->upstream.ssl_certificate->lengths == NULL + && glcf->upstream.ssl_certificate_key->lengths == NULL) { - glcf->upstream.ssl_passwords = - ngx_ssl_preserve_passwords(cf, glcf->upstream.ssl_passwords); - if (glcf->upstream.ssl_passwords == NULL) { - return NGX_ERROR; - } - - } else { if (ngx_ssl_certificate(cf, glcf->upstream.ssl, &glcf->upstream.ssl_certificate->value, &glcf->upstream.ssl_certificate_key->value, diff --git a/src/http/modules/ngx_http_proxy_module.c b/src/http/modules/ngx_http_proxy_module.c index 27c34fef2a5..d4c5abf6258 100644 --- a/src/http/modules/ngx_http_proxy_module.c +++ b/src/http/modules/ngx_http_proxy_module.c @@ -3976,8 +3976,13 @@ ngx_http_proxy_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) prev->upstream.ssl_certificate_key, NULL); ngx_conf_merge_ptr_value(conf->upstream.ssl_certificate_cache, prev->upstream.ssl_certificate_cache, NULL); - ngx_conf_merge_ptr_value(conf->upstream.ssl_passwords, - prev->upstream.ssl_passwords, NULL); + + if (ngx_http_upstream_merge_ssl_passwords(cf, &conf->upstream, + &prev->upstream) + != NGX_OK) + { + return NGX_CONF_ERROR; + } ngx_conf_merge_ptr_value(conf->ssl_conf_commands, prev->ssl_conf_commands, NULL); @@ -5337,16 +5342,9 @@ ngx_http_proxy_set_ssl(ngx_conf_t *cf, ngx_http_proxy_loc_conf_t *plcf) return NGX_ERROR; } - if (plcf->upstream.ssl_certificate->lengths - || plcf->upstream.ssl_certificate_key->lengths) + if (plcf->upstream.ssl_certificate->lengths == NULL + && plcf->upstream.ssl_certificate_key->lengths == NULL) { - plcf->upstream.ssl_passwords = - ngx_ssl_preserve_passwords(cf, plcf->upstream.ssl_passwords); - if (plcf->upstream.ssl_passwords == NULL) { - return NGX_ERROR; - } - - } else { if (ngx_ssl_certificate(cf, plcf->upstream.ssl, &plcf->upstream.ssl_certificate->value, &plcf->upstream.ssl_certificate_key->value, diff --git a/src/http/modules/ngx_http_uwsgi_module.c b/src/http/modules/ngx_http_uwsgi_module.c index 14aae5bf121..51a861d9aeb 100644 --- a/src/http/modules/ngx_http_uwsgi_module.c +++ b/src/http/modules/ngx_http_uwsgi_module.c @@ -1933,8 +1933,13 @@ ngx_http_uwsgi_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) prev->upstream.ssl_certificate_key, NULL); ngx_conf_merge_ptr_value(conf->upstream.ssl_certificate_cache, prev->upstream.ssl_certificate_cache, NULL); - ngx_conf_merge_ptr_value(conf->upstream.ssl_passwords, - prev->upstream.ssl_passwords, NULL); + + if (ngx_http_upstream_merge_ssl_passwords(cf, &conf->upstream, + &prev->upstream) + != NGX_OK) + { + return NGX_CONF_ERROR; + } ngx_conf_merge_ptr_value(conf->ssl_conf_commands, prev->ssl_conf_commands, NULL); @@ -2685,16 +2690,9 @@ ngx_http_uwsgi_set_ssl(ngx_conf_t *cf, ngx_http_uwsgi_loc_conf_t *uwcf) return NGX_ERROR; } - if (uwcf->upstream.ssl_certificate->lengths - || uwcf->upstream.ssl_certificate_key->lengths) + if (uwcf->upstream.ssl_certificate->lengths == NULL + && uwcf->upstream.ssl_certificate_key->lengths == NULL) { - uwcf->upstream.ssl_passwords = - ngx_ssl_preserve_passwords(cf, uwcf->upstream.ssl_passwords); - if (uwcf->upstream.ssl_passwords == NULL) { - return NGX_ERROR; - } - - } else { if (ngx_ssl_certificate(cf, uwcf->upstream.ssl, &uwcf->upstream.ssl_certificate->value, &uwcf->upstream.ssl_certificate_key->value, diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c index 77dc032f290..d4cf1b7fe9f 100644 --- a/src/http/ngx_http_upstream.c +++ b/src/http/ngx_http_upstream.c @@ -6921,6 +6921,61 @@ ngx_http_upstream_hide_headers_hash(ngx_conf_t *cf, } +#if (NGX_HTTP_SSL) + +ngx_int_t +ngx_http_upstream_merge_ssl_passwords(ngx_conf_t *cf, + ngx_http_upstream_conf_t *conf, ngx_http_upstream_conf_t *prev) +{ + ngx_uint_t preserve; + + ngx_conf_merge_ptr_value(conf->ssl_passwords, prev->ssl_passwords, NULL); + + if (conf->ssl_certificate == NULL + || conf->ssl_certificate->value.len == 0 + || conf->ssl_certificate_key == NULL) + { + return NGX_OK; + } + + if (conf->ssl_certificate->lengths == NULL + && conf->ssl_certificate_key->lengths == NULL) + { + if (conf->ssl_passwords && conf->ssl_passwords->pool == NULL) { + /* un-preserve empty password list */ + conf->ssl_passwords = NULL; + } + + return NGX_OK; + } + + if (conf->ssl_passwords && conf->ssl_passwords->pool != cf->temp_pool) { + /* already preserved */ + return NGX_OK; + } + + preserve = (conf->ssl_passwords == prev->ssl_passwords) ? 1 : 0; + + conf->ssl_passwords = ngx_ssl_preserve_passwords(cf, conf->ssl_passwords); + if (conf->ssl_passwords == NULL) { + return NGX_ERROR; + } + + /* + * special handling to keep a preserved ssl_passwords copy + * in the previous configuration to inherit it to all children + */ + + if (preserve) { + prev->ssl_passwords = conf->ssl_passwords; + } + + return NGX_OK; +} + +#endif + + static void * ngx_http_upstream_create_main_conf(ngx_conf_t *cf) { diff --git a/src/http/ngx_http_upstream.h b/src/http/ngx_http_upstream.h index 069c0f7a467..e0a903669ba 100644 --- a/src/http/ngx_http_upstream.h +++ b/src/http/ngx_http_upstream.h @@ -437,6 +437,10 @@ char *ngx_http_upstream_param_set_slot(ngx_conf_t *cf, ngx_command_t *cmd, ngx_int_t ngx_http_upstream_hide_headers_hash(ngx_conf_t *cf, ngx_http_upstream_conf_t *conf, ngx_http_upstream_conf_t *prev, ngx_str_t *default_hide_headers, ngx_hash_init_t *hash); +#if (NGX_HTTP_SSL) +ngx_int_t ngx_http_upstream_merge_ssl_passwords(ngx_conf_t *cf, + ngx_http_upstream_conf_t *conf, ngx_http_upstream_conf_t *prev); +#endif #define ngx_http_conf_upstream_srv_conf(uscf, module) \ diff --git a/src/stream/ngx_stream_proxy_module.c b/src/stream/ngx_stream_proxy_module.c index 7f8bfc4e020..6e51585f6e1 100644 --- a/src/stream/ngx_stream_proxy_module.c +++ b/src/stream/ngx_stream_proxy_module.c @@ -108,6 +108,8 @@ static ngx_int_t ngx_stream_proxy_ssl_name(ngx_stream_session_t *s); static ngx_int_t ngx_stream_proxy_ssl_certificate(ngx_stream_session_t *s); static ngx_int_t ngx_stream_proxy_merge_ssl(ngx_conf_t *cf, ngx_stream_proxy_srv_conf_t *conf, ngx_stream_proxy_srv_conf_t *prev); +static ngx_int_t ngx_stream_proxy_merge_ssl_passwords(ngx_conf_t *cf, + ngx_stream_proxy_srv_conf_t *conf, ngx_stream_proxy_srv_conf_t *prev); static ngx_int_t ngx_stream_proxy_set_ssl(ngx_conf_t *cf, ngx_stream_proxy_srv_conf_t *pscf); @@ -2315,7 +2317,9 @@ ngx_stream_proxy_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) ngx_conf_merge_ptr_value(conf->ssl_certificate_cache, prev->ssl_certificate_cache, NULL); - ngx_conf_merge_ptr_value(conf->ssl_passwords, prev->ssl_passwords, NULL); + if (ngx_stream_proxy_merge_ssl_passwords(cf, conf, prev) != NGX_OK) { + return NGX_CONF_ERROR; + } ngx_conf_merge_ptr_value(conf->ssl_conf_commands, prev->ssl_conf_commands, NULL); @@ -2381,6 +2385,57 @@ ngx_stream_proxy_merge_ssl(ngx_conf_t *cf, ngx_stream_proxy_srv_conf_t *conf, } +static ngx_int_t +ngx_stream_proxy_merge_ssl_passwords(ngx_conf_t *cf, + ngx_stream_proxy_srv_conf_t *conf, ngx_stream_proxy_srv_conf_t *prev) +{ + ngx_uint_t preserve; + + ngx_conf_merge_ptr_value(conf->ssl_passwords, prev->ssl_passwords, NULL); + + if (conf->ssl_certificate == NULL + || conf->ssl_certificate->value.len == 0 + || conf->ssl_certificate_key == NULL) + { + return NGX_OK; + } + + if (conf->ssl_certificate->lengths == NULL + && conf->ssl_certificate_key->lengths == NULL) + { + if (conf->ssl_passwords && conf->ssl_passwords->pool == NULL) { + /* un-preserve empty password list */ + conf->ssl_passwords = NULL; + } + + return NGX_OK; + } + + if (conf->ssl_passwords && conf->ssl_passwords->pool != cf->temp_pool) { + /* already preserved */ + return NGX_OK; + } + + preserve = (conf->ssl_passwords == prev->ssl_passwords) ? 1 : 0; + + conf->ssl_passwords = ngx_ssl_preserve_passwords(cf, conf->ssl_passwords); + if (conf->ssl_passwords == NULL) { + return NGX_ERROR; + } + + /* + * special handling to keep a preserved ssl_passwords copy + * in the previous configuration to inherit it to all children + */ + + if (preserve) { + prev->ssl_passwords = conf->ssl_passwords; + } + + return NGX_OK; +} + + static ngx_int_t ngx_stream_proxy_set_ssl(ngx_conf_t *cf, ngx_stream_proxy_srv_conf_t *pscf) { @@ -2418,16 +2473,9 @@ ngx_stream_proxy_set_ssl(ngx_conf_t *cf, ngx_stream_proxy_srv_conf_t *pscf) return NGX_ERROR; } - if (pscf->ssl_certificate->lengths - || pscf->ssl_certificate_key->lengths) + if (pscf->ssl_certificate->lengths == NULL + && pscf->ssl_certificate_key->lengths == NULL) { - pscf->ssl_passwords = - ngx_ssl_preserve_passwords(cf, pscf->ssl_passwords); - if (pscf->ssl_passwords == NULL) { - return NGX_ERROR; - } - - } else { if (ngx_ssl_certificate(cf, pscf->ssl, &pscf->ssl_certificate->value, &pscf->ssl_certificate_key->value, From b6e7eb0f5792d7a52d2675ee3906e502d63c48e3 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Thu, 3 Apr 2025 15:29:31 +0400 Subject: [PATCH 240/279] SSL: external groups support in $ssl_curve and $ssl_curves. Starting with OpenSSL 3.0, groups may be added externally with pluggable KEM providers. Using SSL_get_negotiated_group(), which makes lookup in a static table with known groups, doesn't allow to list such groups by names leaving them in hex. Adding X25519MLKEM768 to the default group list in OpenSSL 3.5 made this problem more visible. SSL_get0_group_name() and, apparently, SSL_group_to_name() allow to resolve such provider-implemented groups, which is also "generally preferred" over SSL_get_negotiated_group() as documented in OpenSSL git commit 93d4f6133f. This change makes external groups listing by name using SSL_group_to_name() available since OpenSSL 3.0. To preserve "prime256v1" naming for the group 0x0017, and to avoid breaking BoringSSL and older OpenSSL versions support, it is used supplementary for a group that appears to be unknown. See https://github.com/openssl/openssl/issues/27137 for related discussion. --- src/event/ngx_event_openssl.c | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c index 865c78540a3..6992cc4a469 100644 --- a/src/event/ngx_event_openssl.c +++ b/src/event/ngx_event_openssl.c @@ -5040,7 +5040,8 @@ ngx_ssl_get_curve(ngx_connection_t *c, ngx_pool_t *pool, ngx_str_t *s) { #ifdef SSL_get_negotiated_group - int nid; + int nid; + const char *name; nid = SSL_get_negotiated_group(c->ssl->connection); @@ -5052,14 +5053,24 @@ ngx_ssl_get_curve(ngx_connection_t *c, ngx_pool_t *pool, ngx_str_t *s) return NGX_OK; } - s->len = sizeof("0x0000") - 1; +#if (OPENSSL_VERSION_NUMBER >= 0x3000000fL) + name = SSL_group_to_name(c->ssl->connection, nid); +#else + name = NULL; +#endif + s->len = name ? ngx_strlen(name) : sizeof("0x0000") - 1; s->data = ngx_pnalloc(pool, s->len); if (s->data == NULL) { return NGX_ERROR; } - ngx_sprintf(s->data, "0x%04xd", nid & 0xffff); + if (name) { + ngx_memcpy(s->data, name, s->len); + + } else { + ngx_sprintf(s->data, "0x%04xd", nid & 0xffff); + } return NGX_OK; } @@ -5079,6 +5090,7 @@ ngx_ssl_get_curves(ngx_connection_t *c, ngx_pool_t *pool, ngx_str_t *s) int *curves, n, i, nid; u_char *p; size_t len; + const char *name; n = SSL_get1_curves(c->ssl->connection, NULL); @@ -5099,7 +5111,13 @@ ngx_ssl_get_curves(ngx_connection_t *c, ngx_pool_t *pool, ngx_str_t *s) nid = curves[i]; if (nid & TLSEXT_nid_unknown) { - len += sizeof("0x0000") - 1; +#if (OPENSSL_VERSION_NUMBER >= 0x3000000fL) + name = SSL_group_to_name(c->ssl->connection, nid); +#else + name = NULL; +#endif + + len += name ? ngx_strlen(name) : sizeof("0x0000") - 1; } else { len += ngx_strlen(OBJ_nid2sn(nid)); @@ -5119,7 +5137,14 @@ ngx_ssl_get_curves(ngx_connection_t *c, ngx_pool_t *pool, ngx_str_t *s) nid = curves[i]; if (nid & TLSEXT_nid_unknown) { - p = ngx_sprintf(p, "0x%04xd", nid & 0xffff); +#if (OPENSSL_VERSION_NUMBER >= 0x3000000fL) + name = SSL_group_to_name(c->ssl->connection, nid); +#else + name = NULL; +#endif + + p = name ? ngx_cpymem(p, name, ngx_strlen(name)) + : ngx_sprintf(p, "0x%04xd", nid & 0xffff); } else { p = ngx_sprintf(p, "%s", OBJ_nid2sn(nid)); From 2b8b70068a7f7b800ec23390cd2da01b5b91b25f Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Fri, 3 Jan 2025 14:49:47 +0400 Subject: [PATCH 241/279] QUIC: graph-friendly congestion control logging. Improved logging for simpler data extraction for plotting congestion window graphs. In particular, added current milliseconds number from ngx_current_msec. While here, simplified logging text and removed irrelevant data. --- src/event/quic/ngx_event_quic_ack.c | 44 ++++++++++++++++------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/src/event/quic/ngx_event_quic_ack.c b/src/event/quic/ngx_event_quic_ack.c index c953b8042ec..2487ea60d09 100644 --- a/src/event/quic/ngx_event_quic_ack.c +++ b/src/event/quic/ngx_event_quic_ack.c @@ -314,7 +314,7 @@ void ngx_quic_congestion_ack(ngx_connection_t *c, ngx_quic_frame_t *f) { ngx_uint_t blocked; - ngx_msec_t timer; + ngx_msec_t now, timer; ngx_quic_congestion_t *cg; ngx_quic_connection_t *qc; @@ -329,6 +329,8 @@ ngx_quic_congestion_ack(ngx_connection_t *c, ngx_quic_frame_t *f) return; } + now = ngx_current_msec; + blocked = (cg->in_flight >= cg->window) ? 1 : 0; cg->in_flight -= f->plen; @@ -337,8 +339,8 @@ ngx_quic_congestion_ack(ngx_connection_t *c, ngx_quic_frame_t *f) if ((ngx_msec_int_t) timer <= 0) { ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0, - "quic congestion ack recovery win:%uz ss:%z if:%uz", - cg->window, cg->ssthresh, cg->in_flight); + "quic congestion ack rec t:%M win:%uz if:%uz", + now, cg->window, cg->in_flight); goto done; } @@ -346,24 +348,24 @@ ngx_quic_congestion_ack(ngx_connection_t *c, ngx_quic_frame_t *f) if (cg->window < cg->ssthresh) { cg->window += f->plen; - ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0, - "quic congestion slow start win:%uz ss:%z if:%uz", - cg->window, cg->ssthresh, cg->in_flight); + ngx_log_debug4(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic congestion ack ss t:%M win:%uz ss:%z if:%uz", + now, cg->window, cg->ssthresh, cg->in_flight); } else { cg->window += qc->tp.max_udp_payload_size * f->plen / cg->window; ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0, - "quic congestion avoidance win:%uz ss:%z if:%uz", - cg->window, cg->ssthresh, cg->in_flight); + "quic congestion ack reno t:%M win:%uz if:%uz", + now, cg->window, cg->in_flight); } /* prevent recovery_start from wrapping */ - timer = cg->recovery_start - ngx_current_msec + qc->tp.max_idle_timeout * 2; + timer = cg->recovery_start - now + qc->tp.max_idle_timeout * 2; if ((ngx_msec_int_t) timer < 0) { - cg->recovery_start = ngx_current_msec - qc->tp.max_idle_timeout * 2; + cg->recovery_start = now - qc->tp.max_idle_timeout * 2; } done: @@ -541,17 +543,19 @@ ngx_quic_pcg_duration(ngx_connection_t *c) static void ngx_quic_persistent_congestion(ngx_connection_t *c) { + ngx_msec_t now; ngx_quic_congestion_t *cg; ngx_quic_connection_t *qc; qc = ngx_quic_get_connection(c); cg = &qc->congestion; + now = ngx_current_msec; - cg->recovery_start = ngx_current_msec; + cg->recovery_start = now; cg->window = qc->tp.max_udp_payload_size * 2; - ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, - "quic persistent congestion win:%uz", cg->window); + ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic congestion persistent t:%M win:%uz", now, cg->window); } @@ -659,7 +663,7 @@ static void ngx_quic_congestion_lost(ngx_connection_t *c, ngx_quic_frame_t *f) { ngx_uint_t blocked; - ngx_msec_t timer; + ngx_msec_t now, timer; ngx_quic_congestion_t *cg; ngx_quic_connection_t *qc; @@ -681,15 +685,17 @@ ngx_quic_congestion_lost(ngx_connection_t *c, ngx_quic_frame_t *f) timer = f->send_time - cg->recovery_start; + now = ngx_current_msec; + if ((ngx_msec_int_t) timer <= 0) { ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0, - "quic congestion lost recovery win:%uz ss:%z if:%uz", - cg->window, cg->ssthresh, cg->in_flight); + "quic congestion lost rec t:%M win:%uz if:%uz", + now, cg->window, cg->in_flight); goto done; } - cg->recovery_start = ngx_current_msec; + cg->recovery_start = now; cg->window /= 2; if (cg->window < qc->tp.max_udp_payload_size * 2) { @@ -699,8 +705,8 @@ ngx_quic_congestion_lost(ngx_connection_t *c, ngx_quic_frame_t *f) cg->ssthresh = cg->window; ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0, - "quic congestion lost win:%uz ss:%z if:%uz", - cg->window, cg->ssthresh, cg->in_flight); + "quic congestion lost t:%M win:%uz if:%uz", + now, cg->window, cg->in_flight); done: From 3a97111adfb6e538ddef1828bbf04a35a8915c1f Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Tue, 7 Jan 2025 21:14:58 +0400 Subject: [PATCH 242/279] HTTP/3: graceful shutdown on keepalive timeout expiration. Previously, the expiration caused QUIC connection finalization even if there are application-terminated streams finishing sending data. Such finalization terminated these streams. An easy way to trigger this is to request a large file from HTTP/3 over a small MTU. In this case keepalive timeout expiration may abruptly terminate the request stream. --- src/http/v3/ngx_http_v3.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/http/v3/ngx_http_v3.c b/src/http/v3/ngx_http_v3.c index 8db229b29aa..d8597ec5c71 100644 --- a/src/http/v3/ngx_http_v3.c +++ b/src/http/v3/ngx_http_v3.c @@ -70,7 +70,7 @@ ngx_http_v3_keepalive_handler(ngx_event_t *ev) ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "http3 keepalive handler"); - ngx_http_v3_finalize_connection(c, NGX_HTTP_V3_ERR_NO_ERROR, + ngx_http_v3_shutdown_connection(c, NGX_HTTP_V3_ERR_NO_ERROR, "keepalive timeout"); } From 53e7e9eb542fb1d3d885bbca03ed1d704aa08f31 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Fri, 3 Jan 2025 11:17:07 +0400 Subject: [PATCH 243/279] QUIC: use path MTU in congestion window computations. As per RFC 9002, Section B.2, max_datagram_size used in congestion window computations should be based on path MTU. --- src/event/quic/ngx_event_quic.c | 4 ++-- src/event/quic/ngx_event_quic_ack.c | 8 ++++---- src/event/quic/ngx_event_quic_migration.c | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/event/quic/ngx_event_quic.c b/src/event/quic/ngx_event_quic.c index 308597e27ba..70d9748bd2f 100644 --- a/src/event/quic/ngx_event_quic.c +++ b/src/event/quic/ngx_event_quic.c @@ -308,8 +308,8 @@ ngx_quic_new_connection(ngx_connection_t *c, ngx_quic_conf_t *conf, qc->streams.client_max_streams_uni = qc->tp.initial_max_streams_uni; qc->streams.client_max_streams_bidi = qc->tp.initial_max_streams_bidi; - qc->congestion.window = ngx_min(10 * qc->tp.max_udp_payload_size, - ngx_max(2 * qc->tp.max_udp_payload_size, + qc->congestion.window = ngx_min(10 * NGX_QUIC_MIN_INITIAL_SIZE, + ngx_max(2 * NGX_QUIC_MIN_INITIAL_SIZE, 14720)); qc->congestion.ssthresh = (size_t) -1; qc->congestion.recovery_start = ngx_current_msec; diff --git a/src/event/quic/ngx_event_quic_ack.c b/src/event/quic/ngx_event_quic_ack.c index 2487ea60d09..4616e70536f 100644 --- a/src/event/quic/ngx_event_quic_ack.c +++ b/src/event/quic/ngx_event_quic_ack.c @@ -353,7 +353,7 @@ ngx_quic_congestion_ack(ngx_connection_t *c, ngx_quic_frame_t *f) now, cg->window, cg->ssthresh, cg->in_flight); } else { - cg->window += qc->tp.max_udp_payload_size * f->plen / cg->window; + cg->window += (uint64_t) qc->path->mtu * f->plen / cg->window; ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic congestion ack reno t:%M win:%uz if:%uz", @@ -552,7 +552,7 @@ ngx_quic_persistent_congestion(ngx_connection_t *c) now = ngx_current_msec; cg->recovery_start = now; - cg->window = qc->tp.max_udp_payload_size * 2; + cg->window = qc->path->mtu * 2; ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic congestion persistent t:%M win:%uz", now, cg->window); @@ -698,8 +698,8 @@ ngx_quic_congestion_lost(ngx_connection_t *c, ngx_quic_frame_t *f) cg->recovery_start = now; cg->window /= 2; - if (cg->window < qc->tp.max_udp_payload_size * 2) { - cg->window = qc->tp.max_udp_payload_size * 2; + if (cg->window < qc->path->mtu * 2) { + cg->window = qc->path->mtu * 2; } cg->ssthresh = cg->window; diff --git a/src/event/quic/ngx_event_quic_migration.c b/src/event/quic/ngx_event_quic_migration.c index 2d1467e1493..ac22b1327b2 100644 --- a/src/event/quic/ngx_event_quic_migration.c +++ b/src/event/quic/ngx_event_quic_migration.c @@ -182,8 +182,8 @@ ngx_quic_handle_path_response_frame(ngx_connection_t *c, ngx_memzero(&qc->congestion, sizeof(ngx_quic_congestion_t)); - qc->congestion.window = ngx_min(10 * qc->tp.max_udp_payload_size, - ngx_max(2 * qc->tp.max_udp_payload_size, + qc->congestion.window = ngx_min(10 * NGX_QUIC_MIN_INITIAL_SIZE, + ngx_max(2 * NGX_QUIC_MIN_INITIAL_SIZE, 14720)); qc->congestion.ssthresh = (size_t) -1; qc->congestion.recovery_start = ngx_current_msec; From 38236bf74f3e5728eeea488bef381c61842ac1d2 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Fri, 3 Jan 2025 13:01:06 +0400 Subject: [PATCH 244/279] QUIC: prevent spurious congestion control recovery mode. Since recovery_start field was initialized with ngx_current_msec, all congestion events that happened within the same millisecond or cycle iteration, were treated as in recovery mode. Also, when handling persistent congestion, initializing recovery_start with ngx_current_msec resulted in treating all sent packets as in recovery mode, which violates RFC 9002, see example in Appendix B.8. While here, also fixed recovery_start wrap protection. Previously it used 2 * max_idle_timeout time frame for all sent frames, which is not a reliable protection since max_idle_timeout is unrelated to congestion control. Now recovery_start <= now condition is enforced. Note that recovery_start wrap is highly unlikely and can only occur on a 32-bit system if there are no congestion events for 24 days. --- src/event/quic/ngx_event_quic.c | 2 +- src/event/quic/ngx_event_quic_ack.c | 54 ++++++++++++++++++----- src/event/quic/ngx_event_quic_migration.c | 2 +- 3 files changed, 44 insertions(+), 14 deletions(-) diff --git a/src/event/quic/ngx_event_quic.c b/src/event/quic/ngx_event_quic.c index 70d9748bd2f..11497a6d751 100644 --- a/src/event/quic/ngx_event_quic.c +++ b/src/event/quic/ngx_event_quic.c @@ -312,7 +312,7 @@ ngx_quic_new_connection(ngx_connection_t *c, ngx_quic_conf_t *conf, ngx_max(2 * NGX_QUIC_MIN_INITIAL_SIZE, 14720)); qc->congestion.ssthresh = (size_t) -1; - qc->congestion.recovery_start = ngx_current_msec; + qc->congestion.recovery_start = ngx_current_msec - 1; if (pkt->validated && pkt->retried) { qc->tp.retry_scid.len = pkt->dcid.len; diff --git a/src/event/quic/ngx_event_quic_ack.c b/src/event/quic/ngx_event_quic_ack.c index 4616e70536f..29c5bfed19d 100644 --- a/src/event/quic/ngx_event_quic_ack.c +++ b/src/event/quic/ngx_event_quic_ack.c @@ -41,6 +41,7 @@ static ngx_int_t ngx_quic_detect_lost(ngx_connection_t *c, ngx_quic_ack_stat_t *st); static ngx_msec_t ngx_quic_pcg_duration(ngx_connection_t *c); static void ngx_quic_persistent_congestion(ngx_connection_t *c); +static ngx_msec_t ngx_quic_oldest_sent_packet(ngx_connection_t *c); static void ngx_quic_congestion_lost(ngx_connection_t *c, ngx_quic_frame_t *frame); static void ngx_quic_lost_handler(ngx_event_t *ev); @@ -335,6 +336,14 @@ ngx_quic_congestion_ack(ngx_connection_t *c, ngx_quic_frame_t *f) cg->in_flight -= f->plen; + /* prevent recovery_start from wrapping */ + + timer = now - cg->recovery_start; + + if ((ngx_msec_int_t) timer < 0) { + cg->recovery_start = ngx_quic_oldest_sent_packet(c) - 1; + } + timer = f->send_time - cg->recovery_start; if ((ngx_msec_int_t) timer <= 0) { @@ -360,14 +369,6 @@ ngx_quic_congestion_ack(ngx_connection_t *c, ngx_quic_frame_t *f) now, cg->window, cg->in_flight); } - /* prevent recovery_start from wrapping */ - - timer = cg->recovery_start - now + qc->tp.max_idle_timeout * 2; - - if ((ngx_msec_int_t) timer < 0) { - cg->recovery_start = now - qc->tp.max_idle_timeout * 2; - } - done: if (blocked && cg->in_flight < cg->window) { @@ -543,19 +544,48 @@ ngx_quic_pcg_duration(ngx_connection_t *c) static void ngx_quic_persistent_congestion(ngx_connection_t *c) { - ngx_msec_t now; ngx_quic_congestion_t *cg; ngx_quic_connection_t *qc; qc = ngx_quic_get_connection(c); cg = &qc->congestion; - now = ngx_current_msec; - cg->recovery_start = now; + cg->recovery_start = ngx_quic_oldest_sent_packet(c) - 1; cg->window = qc->path->mtu * 2; ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, - "quic congestion persistent t:%M win:%uz", now, cg->window); + "quic congestion persistent t:%M win:%uz", + ngx_current_msec, cg->window); +} + + +static ngx_msec_t +ngx_quic_oldest_sent_packet(ngx_connection_t *c) +{ + ngx_msec_t oldest; + ngx_uint_t i; + ngx_queue_t *q; + ngx_quic_frame_t *start; + ngx_quic_send_ctx_t *ctx; + ngx_quic_connection_t *qc; + + qc = ngx_quic_get_connection(c); + oldest = ngx_current_msec; + + for (i = 0; i < NGX_QUIC_SEND_CTX_LAST; i++) { + ctx = &qc->send_ctx[i]; + + if (!ngx_queue_empty(&ctx->sent)) { + q = ngx_queue_head(&ctx->sent); + start = ngx_queue_data(q, ngx_quic_frame_t, queue); + + if ((ngx_msec_int_t) (start->send_time - oldest) < 0) { + oldest = start->send_time; + } + } + } + + return oldest; } diff --git a/src/event/quic/ngx_event_quic_migration.c b/src/event/quic/ngx_event_quic_migration.c index ac22b1327b2..3caae88e521 100644 --- a/src/event/quic/ngx_event_quic_migration.c +++ b/src/event/quic/ngx_event_quic_migration.c @@ -186,7 +186,7 @@ ngx_quic_handle_path_response_frame(ngx_connection_t *c, ngx_max(2 * NGX_QUIC_MIN_INITIAL_SIZE, 14720)); qc->congestion.ssthresh = (size_t) -1; - qc->congestion.recovery_start = ngx_current_msec; + qc->congestion.recovery_start = ngx_current_msec - 1; ngx_quic_init_rtt(qc); } From 1e883a40db98b70e422ff8e4d1d0e87e3f8ccaa5 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Mon, 10 Mar 2025 12:19:25 +0400 Subject: [PATCH 245/279] QUIC: ngx_msec_t overflow protection. On some systems the value of ngx_current_msec is derived from monotonic clock, for which the following is defined by POSIX: For this clock, the value returned by clock_gettime() represents the amount of time (in seconds and nanoseconds) since an unspecified point in the past. As as result, overflow protection is needed when comparing two ngx_msec_t. The change adds such protection to the ngx_quic_detect_lost() function. --- src/event/quic/ngx_event_quic_ack.c | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/event/quic/ngx_event_quic_ack.c b/src/event/quic/ngx_event_quic_ack.c index 29c5bfed19d..a6f34348b7f 100644 --- a/src/event/quic/ngx_event_quic_ack.c +++ b/src/event/quic/ngx_event_quic_ack.c @@ -449,9 +449,10 @@ ngx_quic_detect_lost(ngx_connection_t *c, ngx_quic_ack_stat_t *st) now = ngx_current_msec; thr = ngx_quic_lost_threshold(qc); - /* send time of lost packets across all send contexts */ - oldest = NGX_TIMER_INFINITE; - newest = NGX_TIMER_INFINITE; +#if (NGX_SUPPRESS_WARN) + oldest = now; + newest = now; +#endif nlost = 0; @@ -484,13 +485,17 @@ ngx_quic_detect_lost(ngx_connection_t *c, ngx_quic_ack_stat_t *st) break; } - if (start->send_time > qc->first_rtt) { + if ((ngx_msec_int_t) (start->send_time - qc->first_rtt) > 0) { - if (oldest == NGX_TIMER_INFINITE || start->send_time < oldest) { + if (nlost == 0 + || (ngx_msec_int_t) (start->send_time - oldest) < 0) + { oldest = start->send_time; } - if (newest == NGX_TIMER_INFINITE || start->send_time > newest) { + if (nlost == 0 + || (ngx_msec_int_t) (start->send_time - newest) > 0) + { newest = start->send_time; } @@ -511,8 +516,9 @@ ngx_quic_detect_lost(ngx_connection_t *c, ngx_quic_ack_stat_t *st) * latest ACK frame. */ - if (st && nlost >= 2 && (st->newest < oldest || st->oldest > newest)) { - + if (st && nlost >= 2 && ((ngx_msec_int_t) (st->newest - oldest) < 0 + || (ngx_msec_int_t) (st->oldest - newest) > 0)) + { if (newest - oldest > ngx_quic_pcg_duration(c)) { ngx_quic_persistent_congestion(c); } From 04c65ccd9a094c00f33bac3a7e0d43cc692409c8 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Sun, 9 Mar 2025 16:09:28 +0400 Subject: [PATCH 246/279] QUIC: all-levels commit and revert functions. Previously, these functions operated on a per-level basis. This however resulted in excessive logging of in_flight and will also led to extra work detecting underutilized congestion window in the followup patches. --- src/event/quic/ngx_event_quic_output.c | 96 ++++++++++++++------------ 1 file changed, 53 insertions(+), 43 deletions(-) diff --git a/src/event/quic/ngx_event_quic_output.c b/src/event/quic/ngx_event_quic_output.c index f087e2bfa6c..9aa7f37bad6 100644 --- a/src/event/quic/ngx_event_quic_output.c +++ b/src/event/quic/ngx_event_quic_output.c @@ -45,9 +45,9 @@ static ngx_int_t ngx_quic_create_datagrams(ngx_connection_t *c); -static void ngx_quic_commit_send(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx); -static void ngx_quic_revert_send(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx, - uint64_t pnum); +static void ngx_quic_commit_send(ngx_connection_t *c); +static void ngx_quic_revert_send(ngx_connection_t *c, + uint64_t preserved_pnum[NGX_QUIC_SEND_CTX_LAST]); #if ((NGX_HAVE_UDP_SEGMENT) && (NGX_HAVE_MSGHDR_MSG_CONTROL)) static ngx_uint_t ngx_quic_allow_segmentation(ngx_connection_t *c); static ngx_int_t ngx_quic_create_segments(ngx_connection_t *c); @@ -127,6 +127,10 @@ ngx_quic_create_datagrams(ngx_connection_t *c) cg = &qc->congestion; path = qc->path; +#if (NGX_SUPPRESS_WARN) + ngx_memzero(preserved_pnum, sizeof(preserved_pnum)); +#endif + while (cg->in_flight < cg->window) { p = dst; @@ -150,12 +154,7 @@ ngx_quic_create_datagrams(ngx_connection_t *c) if (min > len) { /* padding can't be applied - avoid sending the packet */ - - while (i-- > 0) { - ctx = &qc->send_ctx[i]; - ngx_quic_revert_send(c, ctx, preserved_pnum[i]); - } - + ngx_quic_revert_send(c, preserved_pnum); return NGX_OK; } @@ -180,17 +179,12 @@ ngx_quic_create_datagrams(ngx_connection_t *c) } if (n == NGX_AGAIN) { - for (i = 0; i < NGX_QUIC_SEND_CTX_LAST; i++) { - ngx_quic_revert_send(c, &qc->send_ctx[i], preserved_pnum[i]); - } - + ngx_quic_revert_send(c, preserved_pnum); ngx_add_timer(&qc->push, NGX_QUIC_SOCKET_RETRY_DELAY); break; } - for (i = 0; i < NGX_QUIC_SEND_CTX_LAST; i++) { - ngx_quic_commit_send(c, &qc->send_ctx[i]); - } + ngx_quic_commit_send(c); path->sent += len; } @@ -200,31 +194,36 @@ ngx_quic_create_datagrams(ngx_connection_t *c) static void -ngx_quic_commit_send(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx) +ngx_quic_commit_send(ngx_connection_t *c) { + ngx_uint_t i; ngx_queue_t *q; ngx_quic_frame_t *f; + ngx_quic_send_ctx_t *ctx; ngx_quic_congestion_t *cg; ngx_quic_connection_t *qc; qc = ngx_quic_get_connection(c); - cg = &qc->congestion; - while (!ngx_queue_empty(&ctx->sending)) { + for (i = 0; i < NGX_QUIC_SEND_CTX_LAST; i++) { + ctx = &qc->send_ctx[i]; - q = ngx_queue_head(&ctx->sending); - f = ngx_queue_data(q, ngx_quic_frame_t, queue); + while (!ngx_queue_empty(&ctx->sending)) { - ngx_queue_remove(q); + q = ngx_queue_head(&ctx->sending); + f = ngx_queue_data(q, ngx_quic_frame_t, queue); - if (f->pkt_need_ack && !qc->closing) { - ngx_queue_insert_tail(&ctx->sent, q); + ngx_queue_remove(q); - cg->in_flight += f->plen; + if (f->pkt_need_ack && !qc->closing) { + ngx_queue_insert_tail(&ctx->sent, q); - } else { - ngx_quic_free_frame(c, f); + cg->in_flight += f->plen; + + } else { + ngx_quic_free_frame(c, f); + } } } @@ -234,19 +233,30 @@ ngx_quic_commit_send(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx) static void -ngx_quic_revert_send(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx, - uint64_t pnum) +ngx_quic_revert_send(ngx_connection_t *c, uint64_t pnum[NGX_QUIC_SEND_CTX_LAST]) { - ngx_queue_t *q; + ngx_uint_t i; + ngx_queue_t *q; + ngx_quic_send_ctx_t *ctx; + ngx_quic_connection_t *qc; + + qc = ngx_quic_get_connection(c); - while (!ngx_queue_empty(&ctx->sending)) { + for (i = 0; i < NGX_QUIC_SEND_CTX_LAST; i++) { + ctx = &qc->send_ctx[i]; - q = ngx_queue_last(&ctx->sending); - ngx_queue_remove(q); - ngx_queue_insert_head(&ctx->frames, q); - } + if (ngx_queue_empty(&ctx->sending)) { + continue; + } - ctx->pnum = pnum; + do { + q = ngx_queue_last(&ctx->sending); + ngx_queue_remove(q); + ngx_queue_insert_head(&ctx->frames, q); + } while (!ngx_queue_empty(&ctx->sending)); + + ctx->pnum = pnum[i]; + } } @@ -311,13 +321,13 @@ ngx_quic_create_segments(ngx_connection_t *c) size_t len, segsize; ssize_t n; u_char *p, *end; - uint64_t preserved_pnum; - ngx_uint_t nseg; + ngx_uint_t nseg, level; ngx_quic_path_t *path; ngx_quic_send_ctx_t *ctx; ngx_quic_congestion_t *cg; ngx_quic_connection_t *qc; static u_char dst[NGX_QUIC_MAX_UDP_SEGMENT_BUF]; + static uint64_t preserved_pnum[NGX_QUIC_SEND_CTX_LAST]; qc = ngx_quic_get_connection(c); cg = &qc->congestion; @@ -335,7 +345,8 @@ ngx_quic_create_segments(ngx_connection_t *c) nseg = 0; - preserved_pnum = ctx->pnum; + level = ctx - qc->send_ctx; + preserved_pnum[level] = ctx->pnum; for ( ;; ) { @@ -369,19 +380,18 @@ ngx_quic_create_segments(ngx_connection_t *c) } if (n == NGX_AGAIN) { - ngx_quic_revert_send(c, ctx, preserved_pnum); - + ngx_quic_revert_send(c, preserved_pnum); ngx_add_timer(&qc->push, NGX_QUIC_SOCKET_RETRY_DELAY); break; } - ngx_quic_commit_send(c, ctx); + ngx_quic_commit_send(c); path->sent += n; p = dst; nseg = 0; - preserved_pnum = ctx->pnum; + preserved_pnum[level] = ctx->pnum; } } From cd5e4fa1446dff86fafc3b6ffcc11afd527a024f Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Sat, 4 Jan 2025 18:03:46 +0400 Subject: [PATCH 247/279] QUIC: do not increase underutilized congestion window. As per RFC 9002, Section 7.8, congestion window should not be increased when it's underutilized. --- src/event/quic/ngx_event_quic_ack.c | 24 ++++++++++++++++++++++ src/event/quic/ngx_event_quic_ack.h | 1 + src/event/quic/ngx_event_quic_connection.h | 1 + src/event/quic/ngx_event_quic_output.c | 12 ++++++++++- 4 files changed, 37 insertions(+), 1 deletion(-) diff --git a/src/event/quic/ngx_event_quic_ack.c b/src/event/quic/ngx_event_quic_ack.c index a6f34348b7f..bc99947bd44 100644 --- a/src/event/quic/ngx_event_quic_ack.c +++ b/src/event/quic/ngx_event_quic_ack.c @@ -354,6 +354,14 @@ ngx_quic_congestion_ack(ngx_connection_t *c, ngx_quic_frame_t *f) goto done; } + if (cg->idle) { + ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic congestion ack idle t:%M win:%uz if:%uz", + now, cg->window, cg->in_flight); + + goto done; + } + if (cg->window < cg->ssthresh) { cg->window += f->plen; @@ -377,6 +385,22 @@ ngx_quic_congestion_ack(ngx_connection_t *c, ngx_quic_frame_t *f) } +void +ngx_quic_congestion_idle(ngx_connection_t *c, ngx_uint_t idle) +{ + ngx_quic_congestion_t *cg; + ngx_quic_connection_t *qc; + + qc = ngx_quic_get_connection(c); + cg = &qc->congestion; + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic congestion idle:%ui", idle); + + cg->idle = idle; +} + + static void ngx_quic_drop_ack_ranges(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx, uint64_t pn) diff --git a/src/event/quic/ngx_event_quic_ack.h b/src/event/quic/ngx_event_quic_ack.h index 56920c2a5d7..4ad59660fd0 100644 --- a/src/event/quic/ngx_event_quic_ack.h +++ b/src/event/quic/ngx_event_quic_ack.h @@ -17,6 +17,7 @@ ngx_int_t ngx_quic_handle_ack_frame(ngx_connection_t *c, void ngx_quic_congestion_ack(ngx_connection_t *c, ngx_quic_frame_t *frame); +void ngx_quic_congestion_idle(ngx_connection_t *c, ngx_uint_t idle); void ngx_quic_resend_frames(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx); void ngx_quic_set_lost_timer(ngx_connection_t *c); void ngx_quic_pto_handler(ngx_event_t *ev); diff --git a/src/event/quic/ngx_event_quic_connection.h b/src/event/quic/ngx_event_quic_connection.h index 824c92b5791..acc09c1422b 100644 --- a/src/event/quic/ngx_event_quic_connection.h +++ b/src/event/quic/ngx_event_quic_connection.h @@ -169,6 +169,7 @@ typedef struct { size_t window; size_t ssthresh; ngx_msec_t recovery_start; + ngx_uint_t idle; /* unsigned idle:1; */ } ngx_quic_congestion_t; diff --git a/src/event/quic/ngx_event_quic_output.c b/src/event/quic/ngx_event_quic_output.c index 9aa7f37bad6..a92a539f33c 100644 --- a/src/event/quic/ngx_event_quic_output.c +++ b/src/event/quic/ngx_event_quic_output.c @@ -196,7 +196,7 @@ ngx_quic_create_datagrams(ngx_connection_t *c) static void ngx_quic_commit_send(ngx_connection_t *c) { - ngx_uint_t i; + ngx_uint_t i, idle; ngx_queue_t *q; ngx_quic_frame_t *f; ngx_quic_send_ctx_t *ctx; @@ -206,9 +206,15 @@ ngx_quic_commit_send(ngx_connection_t *c) qc = ngx_quic_get_connection(c); cg = &qc->congestion; + idle = 1; + for (i = 0; i < NGX_QUIC_SEND_CTX_LAST; i++) { ctx = &qc->send_ctx[i]; + if (!ngx_queue_empty(&ctx->frames)) { + idle = 0; + } + while (!ngx_queue_empty(&ctx->sending)) { q = ngx_queue_head(&ctx->sending); @@ -229,6 +235,8 @@ ngx_quic_commit_send(ngx_connection_t *c) ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic congestion send if:%uz", cg->in_flight); + + ngx_quic_congestion_idle(c, idle); } @@ -257,6 +265,8 @@ ngx_quic_revert_send(ngx_connection_t *c, uint64_t pnum[NGX_QUIC_SEND_CTX_LAST]) ctx->pnum = pnum[i]; } + + ngx_quic_congestion_idle(c, 1); } From 6bf13e9d57bbc664ac055cdb58c738b09a0f0189 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Mon, 6 Jan 2025 16:27:03 +0400 Subject: [PATCH 248/279] QUIC: do not shrink congestion window after losing an MTU probe. As per RFC 9000, Section 14.4: Loss of a QUIC packet that is carried in a PMTU probe is therefore not a reliable indication of congestion and SHOULD NOT trigger a congestion control reaction. --- src/event/quic/ngx_event_quic_ack.c | 8 ++++++++ src/event/quic/ngx_event_quic_migration.c | 1 + src/event/quic/ngx_event_quic_transport.h | 1 + 3 files changed, 10 insertions(+) diff --git a/src/event/quic/ngx_event_quic_ack.c b/src/event/quic/ngx_event_quic_ack.c index bc99947bd44..d16545a1d36 100644 --- a/src/event/quic/ngx_event_quic_ack.c +++ b/src/event/quic/ngx_event_quic_ack.c @@ -755,6 +755,14 @@ ngx_quic_congestion_lost(ngx_connection_t *c, ngx_quic_frame_t *f) goto done; } + if (f->ignore_loss) { + ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic congestion lost ignore t:%M win:%uz if:%uz", + now, cg->window, cg->in_flight); + + goto done; + } + cg->recovery_start = now; cg->window /= 2; diff --git a/src/event/quic/ngx_event_quic_migration.c b/src/event/quic/ngx_event_quic_migration.c index 3caae88e521..463eeb50335 100644 --- a/src/event/quic/ngx_event_quic_migration.c +++ b/src/event/quic/ngx_event_quic_migration.c @@ -923,6 +923,7 @@ ngx_quic_send_path_mtu_probe(ngx_connection_t *c, ngx_quic_path_t *path) frame->level = ssl_encryption_application; frame->type = NGX_QUIC_FT_PING; + frame->ignore_loss = 1; qc = ngx_quic_get_connection(c); ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_application); diff --git a/src/event/quic/ngx_event_quic_transport.h b/src/event/quic/ngx_event_quic_transport.h index 3e320391afc..dcd763df1ff 100644 --- a/src/event/quic/ngx_event_quic_transport.h +++ b/src/event/quic/ngx_event_quic_transport.h @@ -271,6 +271,7 @@ struct ngx_quic_frame_s { unsigned need_ack:1; unsigned pkt_need_ack:1; unsigned ignore_congestion:1; + unsigned ignore_loss:1; ngx_chain_t *data; union { From a40cc700238796d6668a461e121f6ffee5066394 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Mon, 6 Jan 2025 10:19:56 +0400 Subject: [PATCH 249/279] QUIC: ignore congestion control when sending MTU probes. If connection is network-limited, MTU probes have little chance of being sent since congestion window is almost always full. As a result, PMTUD may not be able to reach the real MTU and the connection may operate with a reduced MTU. The solution is to ignore the congestion window. This may lead to a temporary increase in in-flight count beyond congestion window. --- src/event/quic/ngx_event_quic_migration.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/event/quic/ngx_event_quic_migration.c b/src/event/quic/ngx_event_quic_migration.c index 463eeb50335..1d914ffd83c 100644 --- a/src/event/quic/ngx_event_quic_migration.c +++ b/src/event/quic/ngx_event_quic_migration.c @@ -924,6 +924,7 @@ ngx_quic_send_path_mtu_probe(ngx_connection_t *c, ngx_quic_path_t *path) frame->level = ssl_encryption_application; frame->type = NGX_QUIC_FT_PING; frame->ignore_loss = 1; + frame->ignore_congestion = 1; qc = ngx_quic_get_connection(c); ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_application); From f9a7e7cc11e71b2c62d4c5b9ac4feb7e92913c64 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Thu, 7 Nov 2024 17:25:45 +0400 Subject: [PATCH 250/279] QUIC: CUBIC congestion control. --- src/event/quic/ngx_event_quic.c | 1 + src/event/quic/ngx_event_quic_ack.c | 189 +++++++++++++++++++-- src/event/quic/ngx_event_quic_connection.h | 6 + src/event/quic/ngx_event_quic_migration.c | 1 + 4 files changed, 185 insertions(+), 12 deletions(-) diff --git a/src/event/quic/ngx_event_quic.c b/src/event/quic/ngx_event_quic.c index 11497a6d751..49d30e82acf 100644 --- a/src/event/quic/ngx_event_quic.c +++ b/src/event/quic/ngx_event_quic.c @@ -312,6 +312,7 @@ ngx_quic_new_connection(ngx_connection_t *c, ngx_quic_conf_t *conf, ngx_max(2 * NGX_QUIC_MIN_INITIAL_SIZE, 14720)); qc->congestion.ssthresh = (size_t) -1; + qc->congestion.mtu = NGX_QUIC_MIN_INITIAL_SIZE; qc->congestion.recovery_start = ngx_current_msec - 1; if (pkt->validated && pkt->retried) { diff --git a/src/event/quic/ngx_event_quic_ack.c b/src/event/quic/ngx_event_quic_ack.c index d16545a1d36..6b0eef35e4c 100644 --- a/src/event/quic/ngx_event_quic_ack.c +++ b/src/event/quic/ngx_event_quic_ack.c @@ -20,6 +20,10 @@ /* RFC 9002, 7.6.1. Duration: kPersistentCongestionThreshold */ #define NGX_QUIC_PERSISTENT_CONGESTION_THR 3 +/* CUBIC parameters x10 */ +#define NGX_QUIC_CUBIC_BETA 7 +#define MGX_QUIC_CUBIC_C 4 + /* send time of ACK'ed packets */ typedef struct { @@ -35,10 +39,12 @@ static void ngx_quic_rtt_sample(ngx_connection_t *c, ngx_quic_ack_frame_t *ack, static ngx_int_t ngx_quic_handle_ack_frame_range(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx, uint64_t min, uint64_t max, ngx_quic_ack_stat_t *st); +static size_t ngx_quic_congestion_cubic(ngx_connection_t *c); static void ngx_quic_drop_ack_ranges(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx, uint64_t pn); static ngx_int_t ngx_quic_detect_lost(ngx_connection_t *c, ngx_quic_ack_stat_t *st); +static ngx_msec_t ngx_quic_congestion_cubic_time(ngx_connection_t *c); static ngx_msec_t ngx_quic_pcg_duration(ngx_connection_t *c); static void ngx_quic_persistent_congestion(ngx_connection_t *c); static ngx_msec_t ngx_quic_oldest_sent_packet(ngx_connection_t *c); @@ -314,6 +320,7 @@ ngx_quic_handle_ack_frame_range(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx, void ngx_quic_congestion_ack(ngx_connection_t *c, ngx_quic_frame_t *f) { + size_t w_cubic; ngx_uint_t blocked; ngx_msec_t now, timer; ngx_quic_congestion_t *cg; @@ -370,11 +377,46 @@ ngx_quic_congestion_ack(ngx_connection_t *c, ngx_quic_frame_t *f) now, cg->window, cg->ssthresh, cg->in_flight); } else { - cg->window += (uint64_t) qc->path->mtu * f->plen / cg->window; - ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0, - "quic congestion ack reno t:%M win:%uz if:%uz", - now, cg->window, cg->in_flight); + /* RFC 9438, 4.2. Window Increase Function */ + + w_cubic = ngx_quic_congestion_cubic(c); + + if (cg->window < cg->w_prior) { + cg->w_est += (uint64_t) cg->mtu * f->plen + * 3 * (10 - NGX_QUIC_CUBIC_BETA) + / (10 + NGX_QUIC_CUBIC_BETA) / cg->window; + + } else { + cg->w_est += (uint64_t) cg->mtu * f->plen / cg->window; + } + + if (w_cubic < cg->w_est) { + cg->window = cg->w_est; + + ngx_log_debug4(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic congestion ack reno t:%M win:%uz c:%uz if:%uz", + now, cg->window, w_cubic, cg->in_flight); + + } else if (w_cubic > cg->window) { + + if (w_cubic >= cg->window * 3 / 2) { + cg->window += cg->mtu / 2; + + } else { + cg->window += (uint64_t) cg->mtu * (w_cubic - cg->window) + / cg->window; + } + + ngx_log_debug4(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic congestion ack cubic t:%M win:%uz c:%uz if:%uz", + now, cg->window, w_cubic, cg->in_flight); + + } else { + ngx_log_debug4(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic congestion ack skip t:%M win:%uz c:%uz if:%uz", + now, cg->window, w_cubic, cg->in_flight); + } } done: @@ -385,9 +427,62 @@ ngx_quic_congestion_ack(ngx_connection_t *c, ngx_quic_frame_t *f) } +static size_t +ngx_quic_congestion_cubic(ngx_connection_t *c) +{ + int64_t w, t, cc; + ngx_msec_t now; + ngx_quic_congestion_t *cg; + ngx_quic_connection_t *qc; + + qc = ngx_quic_get_connection(c); + cg = &qc->congestion; + + ngx_quic_congestion_idle(c, cg->idle); + + now = ngx_current_msec; + t = (ngx_msec_int_t) (now - cg->k); + + if (t > 1000000) { + w = NGX_MAX_SIZE_T_VALUE; + goto done; + } + + if (t < -1000000) { + w = 0; + goto done; + } + + /* + * RFC 9438, Figure 1 + * + * w_cubic = C * (t_msec / 1000) ^ 3 * mtu + w_max + */ + + cc = 10000000000ll / (int64_t) cg->mtu / MGX_QUIC_CUBIC_C; + w = t * t * t / cc + (int64_t) cg->w_max; + + if (w > NGX_MAX_SIZE_T_VALUE) { + w = NGX_MAX_SIZE_T_VALUE; + } + + if (w < 0) { + w = 0; + } + +done: + + ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic cubic t:%L w:%L wm:%uz", t, w, cg->w_max); + + return w; +} + + void ngx_quic_congestion_idle(ngx_connection_t *c, ngx_uint_t idle) { + ngx_msec_t now; ngx_quic_congestion_t *cg; ngx_quic_connection_t *qc; @@ -397,6 +492,18 @@ ngx_quic_congestion_idle(ngx_connection_t *c, ngx_uint_t idle) ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic congestion idle:%ui", idle); + if (cg->window >= cg->ssthresh) { + /* RFC 9438, 5.8. Behavior for Application-Limited Flows */ + + now = ngx_current_msec; + + if (cg->idle) { + cg->k += now - cg->idle_start; + } + + cg->idle_start = now; + } + cg->idle = idle; } @@ -580,8 +687,9 @@ ngx_quic_persistent_congestion(ngx_connection_t *c) qc = ngx_quic_get_connection(c); cg = &qc->congestion; + cg->mtu = qc->path->mtu; cg->recovery_start = ngx_quic_oldest_sent_packet(c) - 1; - cg->window = qc->path->mtu * 2; + cg->window = cg->mtu * 2; ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic congestion persistent t:%M win:%uz", @@ -763,14 +871,19 @@ ngx_quic_congestion_lost(ngx_connection_t *c, ngx_quic_frame_t *f) goto done; } - cg->recovery_start = now; - cg->window /= 2; - - if (cg->window < qc->path->mtu * 2) { - cg->window = qc->path->mtu * 2; - } + /* RFC 9438, 4.6. Multiplicative Decrease */ - cg->ssthresh = cg->window; + cg->mtu = qc->path->mtu; + cg->recovery_start = now; + cg->w_prior = cg->window; + /* RFC 9438, 4.7. Fast Convergence */ + cg->w_max = (cg->window < cg->w_max) + ? cg->window * (10 + NGX_QUIC_CUBIC_BETA) / 20 : cg->window; + cg->ssthresh = cg->in_flight * NGX_QUIC_CUBIC_BETA / 10; + cg->window = ngx_max(cg->ssthresh, cg->mtu * 2); + cg->w_est = cg->window; + cg->k = now + ngx_quic_congestion_cubic_time(c); + cg->idle_start = now; ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic congestion lost t:%M win:%uz if:%uz", @@ -784,6 +897,58 @@ ngx_quic_congestion_lost(ngx_connection_t *c, ngx_quic_frame_t *f) } +static ngx_msec_t +ngx_quic_congestion_cubic_time(ngx_connection_t *c) +{ + int64_t v, x, d, cc; + ngx_uint_t n; + ngx_quic_congestion_t *cg; + ngx_quic_connection_t *qc; + + qc = ngx_quic_get_connection(c); + cg = &qc->congestion; + + /* + * RFC 9438, Figure 2 + * + * k_msec = ((w_max - cwnd_epoch) / C / mtu) ^ 1/3 * 1000 + */ + + if (cg->w_max <= cg->window) { + return 0; + } + + cc = 10000000000ll / (int64_t) cg->mtu / MGX_QUIC_CUBIC_C; + v = (int64_t) (cg->w_max - cg->window) * cc; + + /* + * Newton-Raphson method for x ^ 3 = v: + * + * x_next = (2 * x_prev + v / x_prev ^ 2) / 3 + */ + + x = 5000; + + for (n = 1; n <= 10; n++) { + d = (v / x / x - x) / 3; + x += d; + + if (ngx_abs(d) <= 100) { + break; + } + } + + if (x > NGX_MAX_SIZE_T_VALUE) { + return NGX_MAX_SIZE_T_VALUE; + } + + ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic cubic time:%L n:%ui", x, n); + + return x; +} + + void ngx_quic_set_lost_timer(ngx_connection_t *c) { diff --git a/src/event/quic/ngx_event_quic_connection.h b/src/event/quic/ngx_event_quic_connection.h index acc09c1422b..716d62308ea 100644 --- a/src/event/quic/ngx_event_quic_connection.h +++ b/src/event/quic/ngx_event_quic_connection.h @@ -168,7 +168,13 @@ typedef struct { size_t in_flight; size_t window; size_t ssthresh; + size_t w_max; + size_t w_est; + size_t w_prior; + size_t mtu; ngx_msec_t recovery_start; + ngx_msec_t idle_start; + ngx_msec_t k; ngx_uint_t idle; /* unsigned idle:1; */ } ngx_quic_congestion_t; diff --git a/src/event/quic/ngx_event_quic_migration.c b/src/event/quic/ngx_event_quic_migration.c index 1d914ffd83c..6befc34270b 100644 --- a/src/event/quic/ngx_event_quic_migration.c +++ b/src/event/quic/ngx_event_quic_migration.c @@ -186,6 +186,7 @@ ngx_quic_handle_path_response_frame(ngx_connection_t *c, ngx_max(2 * NGX_QUIC_MIN_INITIAL_SIZE, 14720)); qc->congestion.ssthresh = (size_t) -1; + qc->congestion.mtu = NGX_QUIC_MIN_INITIAL_SIZE; qc->congestion.recovery_start = ngx_current_msec - 1; ngx_quic_init_rtt(qc); From 2fb32ff24d431211b673ff9c854352ca0c74e27c Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Fri, 4 Apr 2025 17:39:05 +0400 Subject: [PATCH 251/279] QUIC: optimized connection frame threshold. Previosly the threshold was hardcoded at 10000. This value is too low for high BDP networks. For example, if all frames are STREAM frames, and MTU is 1500, the upper limit for congestion window would be roughly 15M (10000 * 1500). With 100ms RTT it's just a 1.2Gbps network (15M * 10 * 8). In reality, the limit is even lower because of other frame types. Also, the number of frames that could be used simultaneously depends on the total amount of data buffered in all server streams, and client flow control. The change sets frame threshold based on max concurrent streams and stream buffer size, the product of which is the maximum number of in-flight stream data in all server streams at any moment. The value is divided by 2000 to account for a typical MTU 1500 and the fact that not all frames are STREAM frames. --- src/event/quic/ngx_event_quic.c | 4 ++++ src/event/quic/ngx_event_quic_connection.h | 1 + src/event/quic/ngx_event_quic_frames.c | 2 +- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/event/quic/ngx_event_quic.c b/src/event/quic/ngx_event_quic.c index 49d30e82acf..4682ecad9fc 100644 --- a/src/event/quic/ngx_event_quic.c +++ b/src/event/quic/ngx_event_quic.c @@ -315,6 +315,10 @@ ngx_quic_new_connection(ngx_connection_t *c, ngx_quic_conf_t *conf, qc->congestion.mtu = NGX_QUIC_MIN_INITIAL_SIZE; qc->congestion.recovery_start = ngx_current_msec - 1; + qc->max_frames = (conf->max_concurrent_streams_uni + + conf->max_concurrent_streams_bidi) + * conf->stream_buffer_size / 2000; + if (pkt->validated && pkt->retried) { qc->tp.retry_scid.len = pkt->dcid.len; qc->tp.retry_scid.data = ngx_pstrdup(c->pool, &pkt->dcid); diff --git a/src/event/quic/ngx_event_quic_connection.h b/src/event/quic/ngx_event_quic_connection.h index 716d62308ea..04cda859ee0 100644 --- a/src/event/quic/ngx_event_quic_connection.h +++ b/src/event/quic/ngx_event_quic_connection.h @@ -261,6 +261,7 @@ struct ngx_quic_connection_s { ngx_buf_t *free_shadow_bufs; ngx_uint_t nframes; + ngx_uint_t max_frames; #ifdef NGX_QUIC_DEBUG_ALLOC ngx_uint_t nbufs; ngx_uint_t nshadowbufs; diff --git a/src/event/quic/ngx_event_quic_frames.c b/src/event/quic/ngx_event_quic_frames.c index 6ea908cc1ca..888e8bda289 100644 --- a/src/event/quic/ngx_event_quic_frames.c +++ b/src/event/quic/ngx_event_quic_frames.c @@ -214,7 +214,7 @@ ngx_quic_alloc_frame(ngx_connection_t *c) "quic reuse frame n:%ui", qc->nframes); #endif - } else if (qc->nframes < 10000) { + } else if (qc->nframes < qc->max_frames) { frame = ngx_palloc(c->pool, sizeof(ngx_quic_frame_t)); if (frame == NULL) { return NULL; From aa49a416b8ea762558211de25d6ee70ca73bb373 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Mon, 14 Apr 2025 17:16:47 +0400 Subject: [PATCH 252/279] QUIC: dynamic packet threshold. RFC 9002, Section 6.1.1 defines packet reordering threshold as 3. Testing shows that such low value leads to spurious packet losses followed by congestion window collapse. The change implements dynamic packet threshold detection based on in-flight packet range. Packet threshold is defined as half the number of in-flight packets, with mininum value of 3. Also, renamed ngx_quic_lost_threshold() to ngx_quic_time_threshold() for better compliance with RFC 9002 terms. --- src/event/quic/ngx_event_quic_ack.c | 48 +++++++++++++++++++++++------ 1 file changed, 39 insertions(+), 9 deletions(-) diff --git a/src/event/quic/ngx_event_quic_ack.c b/src/event/quic/ngx_event_quic_ack.c index 6b0eef35e4c..b8b72e943df 100644 --- a/src/event/quic/ngx_event_quic_ack.c +++ b/src/event/quic/ngx_event_quic_ack.c @@ -33,7 +33,8 @@ typedef struct { } ngx_quic_ack_stat_t; -static ngx_inline ngx_msec_t ngx_quic_lost_threshold(ngx_quic_connection_t *qc); +static ngx_inline ngx_msec_t ngx_quic_time_threshold(ngx_quic_connection_t *qc); +static uint64_t ngx_quic_packet_threshold(ngx_quic_send_ctx_t *ctx); static void ngx_quic_rtt_sample(ngx_connection_t *c, ngx_quic_ack_frame_t *ack, enum ssl_encryption_level_t level, ngx_msec_t send_time); static ngx_int_t ngx_quic_handle_ack_frame_range(ngx_connection_t *c, @@ -55,7 +56,7 @@ static void ngx_quic_lost_handler(ngx_event_t *ev); /* RFC 9002, 6.1.2. Time Threshold: kTimeThreshold, kGranularity */ static ngx_inline ngx_msec_t -ngx_quic_lost_threshold(ngx_quic_connection_t *qc) +ngx_quic_time_threshold(ngx_quic_connection_t *qc) { ngx_msec_t thr; @@ -66,6 +67,29 @@ ngx_quic_lost_threshold(ngx_quic_connection_t *qc) } +static uint64_t +ngx_quic_packet_threshold(ngx_quic_send_ctx_t *ctx) +{ + uint64_t pkt_thr; + ngx_queue_t *q; + ngx_quic_frame_t *f; + + if (ngx_queue_empty(&ctx->sent)) { + return NGX_QUIC_PKT_THR; + } + + q = ngx_queue_head(&ctx->sent); + f = ngx_queue_data(q, ngx_quic_frame_t, queue); + pkt_thr = (ctx->pnum - f->pnum) / 2; + + if (pkt_thr <= NGX_QUIC_PKT_THR) { + return NGX_QUIC_PKT_THR; + } + + return pkt_thr; +} + + ngx_int_t ngx_quic_handle_ack_frame(ngx_connection_t *c, ngx_quic_header_t *pkt, ngx_quic_frame_t *f) @@ -569,6 +593,7 @@ ngx_quic_drop_ack_ranges(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx, static ngx_int_t ngx_quic_detect_lost(ngx_connection_t *c, ngx_quic_ack_stat_t *st) { + uint64_t pkt_thr; ngx_uint_t i, nlost; ngx_msec_t now, wait, thr, oldest, newest; ngx_queue_t *q; @@ -578,7 +603,7 @@ ngx_quic_detect_lost(ngx_connection_t *c, ngx_quic_ack_stat_t *st) qc = ngx_quic_get_connection(c); now = ngx_current_msec; - thr = ngx_quic_lost_threshold(qc); + thr = ngx_quic_time_threshold(qc); #if (NGX_SUPPRESS_WARN) oldest = now; @@ -595,6 +620,8 @@ ngx_quic_detect_lost(ngx_connection_t *c, ngx_quic_ack_stat_t *st) continue; } + pkt_thr = ngx_quic_packet_threshold(ctx); + while (!ngx_queue_empty(&ctx->sent)) { q = ngx_queue_head(&ctx->sent); @@ -606,12 +633,12 @@ ngx_quic_detect_lost(ngx_connection_t *c, ngx_quic_ack_stat_t *st) wait = start->send_time + thr - now; - ngx_log_debug4(NGX_LOG_DEBUG_EVENT, c->log, 0, - "quic detect_lost pnum:%uL thr:%M wait:%i level:%d", - start->pnum, thr, (ngx_int_t) wait, start->level); + ngx_log_debug5(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic detect_lost pnum:%uL thr:%M pthr:%uL wait:%i level:%d", + start->pnum, thr, pkt_thr, (ngx_int_t) wait, start->level); if ((ngx_msec_int_t) wait > 0 - && ctx->largest_ack - start->pnum < NGX_QUIC_PKT_THR) + && ctx->largest_ack - start->pnum < pkt_thr) { break; } @@ -952,6 +979,7 @@ ngx_quic_congestion_cubic_time(ngx_connection_t *c) void ngx_quic_set_lost_timer(ngx_connection_t *c) { + uint64_t pkt_thr; ngx_uint_t i; ngx_msec_t now; ngx_queue_t *q; @@ -977,10 +1005,12 @@ ngx_quic_set_lost_timer(ngx_connection_t *c) q = ngx_queue_head(&ctx->sent); f = ngx_queue_data(q, ngx_quic_frame_t, queue); w = (ngx_msec_int_t) - (f->send_time + ngx_quic_lost_threshold(qc) - now); + (f->send_time + ngx_quic_time_threshold(qc) - now); if (f->pnum <= ctx->largest_ack) { - if (w < 0 || ctx->largest_ack - f->pnum >= NGX_QUIC_PKT_THR) { + pkt_thr = ngx_quic_packet_threshold(ctx); + + if (w < 0 || ctx->largest_ack - f->pnum >= pkt_thr) { w = 0; } From 6ac8b69f06bc10d5503f636da888fa70095b151c Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Mon, 14 Apr 2025 22:35:27 +0400 Subject: [PATCH 253/279] nginx-1.27.5-RELEASE --- docs/xml/nginx/changes.xml | 71 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/docs/xml/nginx/changes.xml b/docs/xml/nginx/changes.xml index b55177d80b3..ecb063d57a0 100644 --- a/docs/xml/nginx/changes.xml +++ b/docs/xml/nginx/changes.xml @@ -5,6 +5,77 @@ + + + + +контроль перегрузки CUBIC в соединениях QUIC. + + +CUBIC congestion control in QUIC connections. + + + + + +ограничение на максимальный размер кешируемых в разделяемой памяти +SSL-сессий поднято до 8192. + + +the maximum size limit for SSL sessions cached in shared memory +has been raised to 8192. + + + + + +в директивах grpc_ssl_password_file, proxy_ssl_password_file и +uwsgi_ssl_password_file +при загрузке SSL-сертификатов и зашифрованных ключей из переменных; +ошибка появилась в 1.23.1. + + +in the "grpc_ssl_password_file", "proxy_ssl_password_file", and +"uwsgi_ssl_password_file" directives +when loading SSL certificates and encrypted keys from variables; +the bug had appeared in 1.23.1. + + + + + +в переменных $ssl_curve и $ssl_curves +при использовании подключаемых кривых в OpenSSL. + + +in the $ssl_curve and $ssl_curves variables +when using pluggable curves in OpenSSL. + + + + + +nginx не собирался с musl libc.
+Спасибо Piotr Sikora. + + +nginx could not be built with musl libc.
+Thanks to Piotr Sikora. + + + + + +Улучшения производительности и исправления в HTTP/3. + + +Performance improvements and bugfixes in HTTP/3. + + + + + + From 0626e60a754895b0d6e8f6f7d3043c465da7b346 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Wed, 16 Apr 2025 18:48:50 +0400 Subject: [PATCH 254/279] Version bump. --- src/core/nginx.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/nginx.h b/src/core/nginx.h index 0907778535d..72664a531c7 100644 --- a/src/core/nginx.h +++ b/src/core/nginx.h @@ -9,8 +9,8 @@ #define _NGINX_H_INCLUDED_ -#define nginx_version 1027005 -#define NGINX_VERSION "1.27.5" +#define nginx_version 1029000 +#define NGINX_VERSION "1.29.0" #define NGINX_VER "nginx/" NGINX_VERSION #ifdef NGX_BUILD From 04813dac865d538d160de1d01c7248a22503700b Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Tue, 8 Apr 2025 16:54:28 +0400 Subject: [PATCH 255/279] QUIC: lowered log level for unsupported transport parameters. --- src/event/quic/ngx_event_quic_transport.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/event/quic/ngx_event_quic_transport.c b/src/event/quic/ngx_event_quic_transport.c index bb13447b55f..0b3ef4b2ed4 100644 --- a/src/event/quic/ngx_event_quic_transport.c +++ b/src/event/quic/ngx_event_quic_transport.c @@ -1773,7 +1773,7 @@ ngx_quic_parse_transport_params(u_char *p, u_char *end, ngx_quic_tp_t *tp, } if (rc == NGX_DECLINED) { - ngx_log_error(NGX_LOG_INFO, log, 0, + ngx_log_debug2(NGX_LOG_DEBUG_EVENT, log, 0, "quic %s transport param id:0x%xL, skipped", (id % 31 == 27) ? "reserved" : "unknown", id); } From 444954abacef1d77f3dc6e9b1878684c7e6fe5b3 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Wed, 16 Apr 2025 16:56:44 +0400 Subject: [PATCH 256/279] Fixed -Wunterminated-string-initialization with gcc15. --- src/event/quic/ngx_event_quic_protection.c | 12 +++++++----- src/http/v2/ngx_http_v2_filter_module.c | 7 ++++--- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/event/quic/ngx_event_quic_protection.c b/src/event/quic/ngx_event_quic_protection.c index 3f249b36aa2..e5c0df7b442 100644 --- a/src/event/quic/ngx_event_quic_protection.c +++ b/src/event/quic/ngx_event_quic_protection.c @@ -125,9 +125,10 @@ ngx_quic_keys_set_initial_secret(ngx_quic_keys_t *keys, ngx_str_t *secret, ngx_quic_secret_t *client, *server; ngx_quic_ciphers_t ciphers; - static const uint8_t salt[20] = - "\x38\x76\x2c\xf7\xf5\x59\x34\xb3\x4d\x17" - "\x9a\xe6\xa4\xc8\x0c\xad\xcc\xbb\x7f\x0a"; + static const uint8_t salt[20] = { + 0x38, 0x76, 0x2c, 0xf7, 0xf5, 0x59, 0x34, 0xb3, 0x4d, 0x17, + 0x9a, 0xe6, 0xa4, 0xc8, 0x0c, 0xad, 0xcc, 0xbb, 0x7f, 0x0a + }; client = &keys->secrets[ssl_encryption_initial].client; server = &keys->secrets[ssl_encryption_initial].server; @@ -958,8 +959,9 @@ ngx_quic_create_retry_packet(ngx_quic_header_t *pkt, ngx_str_t *res) /* 5.8. Retry Packet Integrity */ static ngx_quic_md_t key = ngx_quic_md( "\xbe\x0c\x69\x0b\x9f\x66\x57\x5a\x1d\x76\x6b\x54\xe3\x68\xc8\x4e"); - static const u_char nonce[NGX_QUIC_IV_LEN] = - "\x46\x15\x99\xd3\x5d\x63\x2b\xf2\x23\x98\x25\xbb"; + static const u_char nonce[NGX_QUIC_IV_LEN] = { + 0x46, 0x15, 0x99, 0xd3, 0x5d, 0x63, 0x2b, 0xf2, 0x23, 0x98, 0x25, 0xbb + }; static ngx_str_t in = ngx_string(""); ad.data = res->data; diff --git a/src/http/v2/ngx_http_v2_filter_module.c b/src/http/v2/ngx_http_v2_filter_module.c index 1e2cafaf145..b63e343a058 100644 --- a/src/http/v2/ngx_http_v2_filter_module.c +++ b/src/http/v2/ngx_http_v2_filter_module.c @@ -115,10 +115,11 @@ ngx_http_v2_header_filter(ngx_http_request_t *r) ngx_http_core_srv_conf_t *cscf; u_char addr[NGX_SOCKADDR_STRLEN]; - static const u_char nginx[5] = "\x84\xaa\x63\x55\xe7"; + static const u_char nginx[5] = { 0x84, 0xaa, 0x63, 0x55, 0xe7 }; #if (NGX_HTTP_GZIP) - static const u_char accept_encoding[12] = - "\x8b\x84\x84\x2d\x69\x5b\x05\x44\x3c\x86\xaa\x6f"; + static const u_char accept_encoding[12] = { + 0x8b, 0x84, 0x84, 0x2d, 0x69, 0x5b, 0x05, 0x44, 0x3c, 0x86, 0xaa, 0x6f + }; #endif static size_t nginx_ver_len = ngx_http_v2_literal_size(NGINX_VER); From 0f9f43b79eed64ab1a876be76ff0f49d499784fc Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Fri, 18 Apr 2025 11:16:57 +0400 Subject: [PATCH 257/279] HTTP/3: fixed NGX_HTTP_V3_VARLEN_INT_LEN value. After fixing ngx_http_v3_encode_varlen_int() in 400eb1b628, NGX_HTTP_V3_VARLEN_INT_LEN retained the old value of 4, which is insufficient for the values over 1073741823 (1G - 1). The NGX_HTTP_V3_VARLEN_INT_LEN macro is used in ngx_http_v3_uni.c to format stream and frame types. Old buffer size is enough for formatting this data. Also, the macro is used in ngx_http_v3_filter_module.c to format output chunks and trailers. Considering output_buffers and proxy_buffer_size are below 1G in all realistic scenarios, the old buffer size is enough here as well. --- src/http/v3/ngx_http_v3.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/http/v3/ngx_http_v3.h b/src/http/v3/ngx_http_v3.h index 9dcb5e6a7dd..8fd212c1f08 100644 --- a/src/http/v3/ngx_http_v3.h +++ b/src/http/v3/ngx_http_v3.h @@ -23,7 +23,7 @@ #define NGX_HTTP_V3_HQ_ALPN_PROTO "\x0Ahq-interop" #define NGX_HTTP_V3_HQ_PROTO "hq-interop" -#define NGX_HTTP_V3_VARLEN_INT_LEN 4 +#define NGX_HTTP_V3_VARLEN_INT_LEN 8 #define NGX_HTTP_V3_PREFIX_INT_LEN 11 #define NGX_HTTP_V3_STREAM_CONTROL 0x00 From b9d0ba6677ff7761c85f5556776d6a6c2a7a7051 Mon Sep 17 00:00:00 2001 From: Aleksei Bavshin Date: Tue, 14 Jan 2025 10:32:24 -0800 Subject: [PATCH 258/279] Core: improved NGX_ALIGNMENT detection on some x86_64 platforms. Previously, the default pool alignment used sizeof(unsigned long), with the expectation that this would match to a platform word size. Certain 64-bit platforms prove this assumption wrong by keeping the 32-bit long type, which is fully compliant with the C standard. This introduces a possibility of suboptimal misaligned access to the data allocated with ngx_palloc() on the affected platforms, which is addressed here by changing the default NGX_ALIGNMENT to a pointer size. As we override the detection in auto/os/conf for all the machine types except x86, and Unix-like 64-bit systems prefer the 64-bit long, the impact of the change should be limited to Win64 x64. --- src/core/ngx_config.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/ngx_config.h b/src/core/ngx_config.h index 1861be60139..707ab216be1 100644 --- a/src/core/ngx_config.h +++ b/src/core/ngx_config.h @@ -94,7 +94,7 @@ typedef intptr_t ngx_flag_t; #ifndef NGX_ALIGNMENT -#define NGX_ALIGNMENT sizeof(unsigned long) /* platform word */ +#define NGX_ALIGNMENT sizeof(uintptr_t) /* platform word */ #endif #define ngx_align(d, a) (((d) + (a - 1)) & ~(a - 1)) From 020b1db7eb187d4a9a5f1d6154c664a463473b36 Mon Sep 17 00:00:00 2001 From: Aleksei Bavshin Date: Tue, 14 Jan 2025 11:11:28 -0800 Subject: [PATCH 259/279] Win32: added detection of ARM64 target. This extends the target selection implemented in dad6ec3aa63f to support Windows ARM64 platforms. OpenSSL support for VC-WIN64-ARM target first appeared in 1.1.1 and is present in all currently supported (3.x) branches. As a side effect, ARM64 Windows builds will get 16-byte alignment along with the rest of non-x86 platforms. This is safe, as malloc on 64-bit Windows guarantees the fundamental alignment of allocations, 16 bytes. --- auto/cc/msvc | 4 ++++ auto/lib/openssl/make | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/auto/cc/msvc b/auto/cc/msvc index 567bac7bcd2..fe7c34e49cd 100644 --- a/auto/cc/msvc +++ b/auto/cc/msvc @@ -26,6 +26,10 @@ ngx_msvc_ver=`echo $NGX_MSVC_VER | sed -e 's/^\([0-9]*\).*/\1/'` case "$NGX_MSVC_VER" in + *ARM64) + NGX_MACHINE=arm64 + ;; + *x64) NGX_MACHINE=amd64 ;; diff --git a/auto/lib/openssl/make b/auto/lib/openssl/make index a7e9369e7d0..f8480146dec 100644 --- a/auto/lib/openssl/make +++ b/auto/lib/openssl/make @@ -13,6 +13,10 @@ case "$CC" in OPENSSL_TARGET=VC-WIN64A ;; + arm64) + OPENSSL_TARGET=VC-WIN64-ARM + ;; + *) OPENSSL_TARGET=VC-WIN32 ;; From 9785db9bd504ff25c1d84857505e6546fc04ae68 Mon Sep 17 00:00:00 2001 From: nandsky Date: Fri, 18 Apr 2025 11:45:12 +0800 Subject: [PATCH 260/279] QUIC: fixed a typo. --- src/event/quic/ngx_event_quic_ack.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/event/quic/ngx_event_quic_ack.c b/src/event/quic/ngx_event_quic_ack.c index b8b72e943df..c7fd96c2cdd 100644 --- a/src/event/quic/ngx_event_quic_ack.c +++ b/src/event/quic/ngx_event_quic_ack.c @@ -22,7 +22,7 @@ /* CUBIC parameters x10 */ #define NGX_QUIC_CUBIC_BETA 7 -#define MGX_QUIC_CUBIC_C 4 +#define NGX_QUIC_CUBIC_C 4 /* send time of ACK'ed packets */ @@ -483,7 +483,7 @@ ngx_quic_congestion_cubic(ngx_connection_t *c) * w_cubic = C * (t_msec / 1000) ^ 3 * mtu + w_max */ - cc = 10000000000ll / (int64_t) cg->mtu / MGX_QUIC_CUBIC_C; + cc = 10000000000ll / (int64_t) cg->mtu / NGX_QUIC_CUBIC_C; w = t * t * t / cc + (int64_t) cg->w_max; if (w > NGX_MAX_SIZE_T_VALUE) { @@ -945,7 +945,7 @@ ngx_quic_congestion_cubic_time(ngx_connection_t *c) return 0; } - cc = 10000000000ll / (int64_t) cg->mtu / MGX_QUIC_CUBIC_C; + cc = 10000000000ll / (int64_t) cg->mtu / NGX_QUIC_CUBIC_C; v = (int64_t) (cg->w_max - cg->window) * cc; /* From 4f8bc0b282b976eb7d044c81cad06e7e6d64e5ff Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Wed, 16 Apr 2025 20:50:29 +0400 Subject: [PATCH 261/279] SSL: fixed build with OPENSSL_NO_DEPRECATED. --- src/event/ngx_event_openssl.h | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h index b7aaaca7512..9ba21a81064 100644 --- a/src/event/ngx_event_openssl.h +++ b/src/event/ngx_event_openssl.h @@ -83,6 +83,17 @@ #endif +#ifdef OPENSSL_NO_DEPRECATED_3_4 +#define SSL_SESSION_get_time(s) SSL_SESSION_get_time_ex(s) +#define SSL_SESSION_set_time(s, t) SSL_SESSION_set_time_ex(s, t) +#endif + + +#ifdef OPENSSL_NO_DEPRECATED_3_0 +#define EVP_CIPHER_CTX_cipher(c) EVP_CIPHER_CTX_get0_cipher(c) +#endif + + typedef struct ngx_ssl_ocsp_s ngx_ssl_ocsp_t; From adda7041582d8565ee1e5e7dfe740db85398e1ce Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Wed, 16 Apr 2025 20:58:57 +0400 Subject: [PATCH 262/279] SSL: fixed build with OPENSSL_NO_DH. --- src/event/ngx_event_openssl.c | 4 ++++ src/event/ngx_event_openssl.h | 2 ++ 2 files changed, 6 insertions(+) diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c index 6992cc4a469..7eb05209dd6 100644 --- a/src/event/ngx_event_openssl.c +++ b/src/event/ngx_event_openssl.c @@ -1315,6 +1315,8 @@ ngx_ssl_passwords_cleanup(void *data) ngx_int_t ngx_ssl_dhparam(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *file) { +#ifndef OPENSSL_NO_DH + BIO *bio; if (file->len == 0) { @@ -1385,6 +1387,8 @@ ngx_ssl_dhparam(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *file) BIO_free(bio); +#endif + return NGX_OK; } diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h index 9ba21a81064..d4a62b82a66 100644 --- a/src/event/ngx_event_openssl.h +++ b/src/event/ngx_event_openssl.h @@ -19,7 +19,9 @@ #include #include #include +#ifndef OPENSSL_NO_DH #include +#endif #ifndef OPENSSL_NO_ENGINE #include #endif From f3542500b6d74e3e88fc84b88144afe67882d1fa Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Fri, 25 Apr 2025 23:32:24 +0400 Subject: [PATCH 263/279] QUIC: do not block ACKs by congestion control. Previously, it was not possible to send acknowledgments if the congestion window was limited or temporarily exceeded, such as after sending a large response or MTU probe. If ACKs were not received from the peer for some reason to update the in-flight bytes counter below the congestion window, this might result in a stalled connection. The fix is to send ACKs regardless of congestion control. This meets RFC 9002, Section 7: : Similar to TCP, packets containing only ACK frames do not count : toward bytes in flight and are not congestion controlled. This is a simplified implementation to send ACK frames from the head of the queue. This was made possible after 6f5f17358. Reported in trac ticket #2621 and subsequently by Vladimir Homutov: https://mailman.nginx.org/pipermail/nginx-devel/2025-April/ZKBAWRJVQXSZ2ISG3YJAF3EWMDRDHCMO.html --- src/event/quic/ngx_event_quic_output.c | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/src/event/quic/ngx_event_quic_output.c b/src/event/quic/ngx_event_quic_output.c index a92a539f33c..01f1f911324 100644 --- a/src/event/quic/ngx_event_quic_output.c +++ b/src/event/quic/ngx_event_quic_output.c @@ -55,7 +55,8 @@ static ssize_t ngx_quic_send_segments(ngx_connection_t *c, u_char *buf, size_t len, struct sockaddr *sockaddr, socklen_t socklen, size_t segment); #endif static ssize_t ngx_quic_output_packet(ngx_connection_t *c, - ngx_quic_send_ctx_t *ctx, u_char *data, size_t max, size_t min); + ngx_quic_send_ctx_t *ctx, u_char *data, size_t max, size_t min, + ngx_uint_t ack_only); static void ngx_quic_init_packet(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx, ngx_quic_header_t *pkt, ngx_quic_path_t *path); static ngx_uint_t ngx_quic_get_padding_level(ngx_connection_t *c); @@ -131,8 +132,7 @@ ngx_quic_create_datagrams(ngx_connection_t *c) ngx_memzero(preserved_pnum, sizeof(preserved_pnum)); #endif - while (cg->in_flight < cg->window) { - + do { p = dst; len = ngx_quic_path_limit(c, path, path->mtu); @@ -158,7 +158,8 @@ ngx_quic_create_datagrams(ngx_connection_t *c) return NGX_OK; } - n = ngx_quic_output_packet(c, ctx, p, len, min); + n = ngx_quic_output_packet(c, ctx, p, len, min, + cg->in_flight >= cg->window); if (n == NGX_ERROR) { return NGX_ERROR; } @@ -187,7 +188,8 @@ ngx_quic_create_datagrams(ngx_connection_t *c) ngx_quic_commit_send(c); path->sent += len; - } + + } while (cg->in_flight < cg->window); return NGX_OK; } @@ -315,6 +317,10 @@ ngx_quic_allow_segmentation(ngx_connection_t *c) bytes += f->len; + if (qc->congestion.in_flight + bytes >= qc->congestion.window) { + return 0; + } + if (bytes > len * 3) { /* require at least ~3 full packets to batch */ return 1; @@ -364,7 +370,7 @@ ngx_quic_create_segments(ngx_connection_t *c) if (len && cg->in_flight + (p - dst) < cg->window) { - n = ngx_quic_output_packet(c, ctx, p, len, len); + n = ngx_quic_output_packet(c, ctx, p, len, len, 0); if (n == NGX_ERROR) { return NGX_ERROR; } @@ -521,7 +527,7 @@ ngx_quic_get_padding_level(ngx_connection_t *c) static ssize_t ngx_quic_output_packet(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx, - u_char *data, size_t max, size_t min) + u_char *data, size_t max, size_t min, ngx_uint_t ack_only) { size_t len, pad, min_payload, max_payload; u_char *p; @@ -585,6 +591,10 @@ ngx_quic_output_packet(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx, { f = ngx_queue_data(q, ngx_quic_frame_t, queue); + if (ack_only && f->type != NGX_QUIC_FT_ACK) { + break; + } + if (len >= max_payload) { break; } From 335993154c85daf4be6bc5b465ff0d4f319082ad Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Tue, 6 May 2025 19:57:44 +0400 Subject: [PATCH 264/279] QUIC: removed level field from ngx_quic_compat_record_t. It was made unused in d15f8f2 after introducing reusable crypto contexts. --- src/event/quic/ngx_event_quic_openssl_compat.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/event/quic/ngx_event_quic_openssl_compat.c b/src/event/quic/ngx_event_quic_openssl_compat.c index 6052bc683ba..a4a8ea1b624 100644 --- a/src/event/quic/ngx_event_quic_openssl_compat.c +++ b/src/event/quic/ngx_event_quic_openssl_compat.c @@ -35,8 +35,6 @@ typedef struct { ngx_str_t payload; uint64_t number; ngx_quic_compat_keys_t *keys; - - enum ssl_encryption_level_t level; } ngx_quic_compat_record_t; @@ -501,7 +499,6 @@ SSL_provide_quic_data(SSL *ssl, enum ssl_encryption_level_t level, rec.log = c->log; rec.number = com->read_record++; rec.keys = &com->keys; - rec.level = level; if (level == ssl_encryption_initial) { n = ngx_min(len, 65535); From 39b1e3fe9de7966c2c66e228cf7b205616de6ee2 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Tue, 6 May 2025 15:53:49 +0400 Subject: [PATCH 265/279] QUIC: removed excessive casts for ngx_ssl_get_connection(). They were blindly copied from ngx_ssl_info_callback(), where the ngx_ssl_conn_t pointer is passed with const qualifier. --- src/event/quic/ngx_event_quic_ssl.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/event/quic/ngx_event_quic_ssl.c b/src/event/quic/ngx_event_quic_ssl.c index ba0b5929fde..6ad3c0e9373 100644 --- a/src/event/quic/ngx_event_quic_ssl.c +++ b/src/event/quic/ngx_event_quic_ssl.c @@ -57,7 +57,7 @@ ngx_quic_set_read_secret(ngx_ssl_conn_t *ssl_conn, ngx_connection_t *c; ngx_quic_connection_t *qc; - c = ngx_ssl_get_connection((ngx_ssl_conn_t *) ssl_conn); + c = ngx_ssl_get_connection(ssl_conn); qc = ngx_quic_get_connection(c); ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, @@ -87,7 +87,7 @@ ngx_quic_set_write_secret(ngx_ssl_conn_t *ssl_conn, ngx_connection_t *c; ngx_quic_connection_t *qc; - c = ngx_ssl_get_connection((ngx_ssl_conn_t *) ssl_conn); + c = ngx_ssl_get_connection(ssl_conn); qc = ngx_quic_get_connection(c); ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, @@ -119,7 +119,7 @@ ngx_quic_set_encryption_secrets(ngx_ssl_conn_t *ssl_conn, const SSL_CIPHER *cipher; ngx_quic_connection_t *qc; - c = ngx_ssl_get_connection((ngx_ssl_conn_t *) ssl_conn); + c = ngx_ssl_get_connection(ssl_conn); qc = ngx_quic_get_connection(c); ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, @@ -180,7 +180,7 @@ ngx_quic_add_handshake_data(ngx_ssl_conn_t *ssl_conn, const unsigned char *alpn_data; #endif - c = ngx_ssl_get_connection((ngx_ssl_conn_t *) ssl_conn); + c = ngx_ssl_get_connection(ssl_conn); qc = ngx_quic_get_connection(c); ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, @@ -279,7 +279,7 @@ ngx_quic_flush_flight(ngx_ssl_conn_t *ssl_conn) #if (NGX_DEBUG) ngx_connection_t *c; - c = ngx_ssl_get_connection((ngx_ssl_conn_t *) ssl_conn); + c = ngx_ssl_get_connection(ssl_conn); ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic ngx_quic_flush_flight()"); @@ -295,7 +295,7 @@ ngx_quic_send_alert(ngx_ssl_conn_t *ssl_conn, enum ssl_encryption_level_t level, ngx_connection_t *c; ngx_quic_connection_t *qc; - c = ngx_ssl_get_connection((ngx_ssl_conn_t *) ssl_conn); + c = ngx_ssl_get_connection(ssl_conn); ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic ngx_quic_send_alert() level:%s alert:%d", From aa43385ffa9acda2f9a45265bbe8e9213d9a840e Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Tue, 6 May 2025 15:17:44 +0400 Subject: [PATCH 266/279] QUIC: removed ALPN feature test. ALPN support is present in all libraries that have QUIC support, it is safe to compile it unconditionally. --- src/event/quic/ngx_event_quic_ssl.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/event/quic/ngx_event_quic_ssl.c b/src/event/quic/ngx_event_quic_ssl.c index 6ad3c0e9373..f255f77a268 100644 --- a/src/event/quic/ngx_event_quic_ssl.c +++ b/src/event/quic/ngx_event_quic_ssl.c @@ -169,16 +169,14 @@ ngx_quic_add_handshake_data(ngx_ssl_conn_t *ssl_conn, u_char *p, *end; size_t client_params_len; ngx_chain_t *out; + unsigned int alpn_len; const uint8_t *client_params; ngx_quic_tp_t ctp; ngx_quic_frame_t *frame; ngx_connection_t *c; + const unsigned char *alpn_data; ngx_quic_send_ctx_t *ctx; ngx_quic_connection_t *qc; -#if defined(TLSEXT_TYPE_application_layer_protocol_negotiation) - unsigned int alpn_len; - const unsigned char *alpn_data; -#endif c = ngx_ssl_get_connection(ssl_conn); qc = ngx_quic_get_connection(c); @@ -193,8 +191,6 @@ ngx_quic_add_handshake_data(ngx_ssl_conn_t *ssl_conn, * here; */ -#if defined(TLSEXT_TYPE_application_layer_protocol_negotiation) - SSL_get0_alpn_selected(ssl_conn, &alpn_data, &alpn_len); if (alpn_len == 0) { @@ -206,8 +202,6 @@ ngx_quic_add_handshake_data(ngx_ssl_conn_t *ssl_conn, return 0; } -#endif - SSL_get_peer_quic_transport_params(ssl_conn, &client_params, &client_params_len); From ef9cd3214ff3d9d1513da8927423f42184dcf8e5 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Tue, 6 May 2025 15:09:28 +0400 Subject: [PATCH 267/279] QUIC: logging level of handshake errors. Various errors reported by SSL_do_handshake() are now logged at the "info" or "crit" level, akin to handshakes on regular TCP connections. --- src/event/ngx_event_openssl.c | 4 +--- src/event/ngx_event_openssl.h | 2 ++ src/event/quic/ngx_event_quic_ssl.c | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c index 7eb05209dd6..a7b38944462 100644 --- a/src/event/ngx_event_openssl.c +++ b/src/event/ngx_event_openssl.c @@ -45,8 +45,6 @@ static ssize_t ngx_ssl_sendfile(ngx_connection_t *c, ngx_buf_t *file, size_t size); static void ngx_ssl_read_handler(ngx_event_t *rev); static void ngx_ssl_shutdown_handler(ngx_event_t *ev); -static void ngx_ssl_connection_error(ngx_connection_t *c, int sslerr, - ngx_err_t err, char *text); static void ngx_ssl_clear_error(ngx_log_t *log); static ngx_int_t ngx_ssl_session_id_context(ngx_ssl_t *ssl, @@ -3301,7 +3299,7 @@ ngx_ssl_shutdown_handler(ngx_event_t *ev) } -static void +void ngx_ssl_connection_error(ngx_connection_t *c, int sslerr, ngx_err_t err, char *text) { diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h index d4a62b82a66..9e68deb4432 100644 --- a/src/event/ngx_event_openssl.h +++ b/src/event/ngx_event_openssl.h @@ -361,6 +361,8 @@ ngx_chain_t *ngx_ssl_send_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit); void ngx_ssl_free_buffer(ngx_connection_t *c); ngx_int_t ngx_ssl_shutdown(ngx_connection_t *c); +void ngx_ssl_connection_error(ngx_connection_t *c, int sslerr, ngx_err_t err, + char *text); void ngx_cdecl ngx_ssl_error(ngx_uint_t level, ngx_log_t *log, ngx_err_t err, char *fmt, ...); void ngx_ssl_cleanup_ctx(void *data); diff --git a/src/event/quic/ngx_event_quic_ssl.c b/src/event/quic/ngx_event_quic_ssl.c index f255f77a268..ddc6c7c3bf9 100644 --- a/src/event/quic/ngx_event_quic_ssl.c +++ b/src/event/quic/ngx_event_quic_ssl.c @@ -427,7 +427,7 @@ ngx_quic_crypto_input(ngx_connection_t *c, ngx_chain_t *data, return NGX_ERROR; } - ngx_ssl_error(NGX_LOG_ERR, c->log, 0, "SSL_do_handshake() failed"); + ngx_ssl_connection_error(c, sslerr, 0, "SSL_do_handshake() failed"); return NGX_ERROR; } } From 47f96993f669543c6cb4979dd3f680ad01314ee5 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Wed, 21 May 2025 19:55:31 +0400 Subject: [PATCH 268/279] QUIC: logging of SSL library errors. Logging level for such errors, which should not normally happen, is changed to NGX_LOG_ALERT, and ngx_log_error() is replaced with ngx_ssl_error() for consistency with the rest of the code. --- src/event/quic/ngx_event_quic_ssl.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/event/quic/ngx_event_quic_ssl.c b/src/event/quic/ngx_event_quic_ssl.c index ddc6c7c3bf9..4f7060ce478 100644 --- a/src/event/quic/ngx_event_quic_ssl.c +++ b/src/event/quic/ngx_event_quic_ssl.c @@ -402,7 +402,7 @@ ngx_quic_crypto_input(ngx_connection_t *c, ngx_chain_t *data, b = cl->buf; if (!SSL_provide_quic_data(ssl_conn, level, b->pos, b->last - b->pos)) { - ngx_ssl_error(NGX_LOG_INFO, c->log, 0, + ngx_ssl_error(NGX_LOG_ALERT, c->log, 0, "SSL_provide_quic_data() failed"); return NGX_ERROR; } @@ -531,7 +531,7 @@ ngx_quic_init_connection(ngx_connection_t *c) } if (SSL_set_quic_method(ssl_conn, &quic_method) == 0) { - ngx_log_error(NGX_LOG_INFO, c->log, 0, + ngx_ssl_error(NGX_LOG_ALERT, c->log, 0, "quic SSL_set_quic_method() failed"); return NGX_ERROR; } @@ -572,14 +572,14 @@ ngx_quic_init_connection(ngx_connection_t *c) #endif if (SSL_set_quic_transport_params(ssl_conn, p, len) == 0) { - ngx_log_error(NGX_LOG_INFO, c->log, 0, + ngx_ssl_error(NGX_LOG_ALERT, c->log, 0, "quic SSL_set_quic_transport_params() failed"); return NGX_ERROR; } #ifdef OPENSSL_IS_BORINGSSL if (SSL_set_quic_early_data_context(ssl_conn, p, clen) == 0) { - ngx_log_error(NGX_LOG_INFO, c->log, 0, + ngx_ssl_error(NGX_LOG_ALERT, c->log, 0, "quic SSL_set_quic_early_data_context() failed"); return NGX_ERROR; } From 7468a10b62276be4adee0fcd6aaf6244270984ab Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Tue, 13 May 2025 20:12:10 +0400 Subject: [PATCH 269/279] QUIC: adjusted handling of callback errors. Changed handshake callbacks to always return success. This allows to avoid logging SSL_do_handshake() errors with empty or cryptic "internal error" OpenSSL error messages at the inappropriate "crit" log level. Further, connections with failed callbacks are closed now right away when using OpenSSL compat layer. This change supersedes and reverts c37fdcdd1, with the conditions to check callbacks invocation kept to slightly improve code readability of control flow; they are optimized out in the resulting assembly code. --- src/event/quic/ngx_event_quic.c | 3 +++ .../quic/ngx_event_quic_openssl_compat.c | 8 ++---- src/event/quic/ngx_event_quic_ssl.c | 27 ++++++++++++------- 3 files changed, 22 insertions(+), 16 deletions(-) diff --git a/src/event/quic/ngx_event_quic.c b/src/event/quic/ngx_event_quic.c index 4682ecad9fc..a4ad85d56bb 100644 --- a/src/event/quic/ngx_event_quic.c +++ b/src/event/quic/ngx_event_quic.c @@ -135,6 +135,9 @@ ngx_quic_apply_transport_params(ngx_connection_t *c, ngx_quic_tp_t *ctp) if (scid.len != ctp->initial_scid.len || ngx_memcmp(scid.data, ctp->initial_scid.data, scid.len) != 0) { + qc->error = NGX_QUIC_ERR_TRANSPORT_PARAMETER_ERROR; + qc->error_reason = "invalid initial_source_connection_id"; + ngx_log_error(NGX_LOG_INFO, c->log, 0, "quic client initial_source_connection_id mismatch"); return NGX_ERROR; diff --git a/src/event/quic/ngx_event_quic_openssl_compat.c b/src/event/quic/ngx_event_quic_openssl_compat.c index a4a8ea1b624..c5762f15521 100644 --- a/src/event/quic/ngx_event_quic_openssl_compat.c +++ b/src/event/quic/ngx_event_quic_openssl_compat.c @@ -437,7 +437,7 @@ ngx_quic_compat_message_callback(int write_p, int version, int content_type, ngx_quic_level_name(level), len); if (com->method->add_handshake_data(ssl, level, buf, len) != 1) { - goto failed; + return; } break; @@ -451,7 +451,7 @@ ngx_quic_compat_message_callback(int write_p, int version, int content_type, ngx_quic_level_name(level), alert, len); if (com->method->send_alert(ssl, level, alert) != 1) { - goto failed; + return; } } @@ -459,10 +459,6 @@ ngx_quic_compat_message_callback(int write_p, int version, int content_type, } return; - -failed: - - ngx_post_event(&qc->close, &ngx_posted_events); } diff --git a/src/event/quic/ngx_event_quic_ssl.c b/src/event/quic/ngx_event_quic_ssl.c index 4f7060ce478..dd7ee3702fb 100644 --- a/src/event/quic/ngx_event_quic_ssl.c +++ b/src/event/quic/ngx_event_quic_ssl.c @@ -72,7 +72,7 @@ ngx_quic_set_read_secret(ngx_ssl_conn_t *ssl_conn, cipher, rsecret, secret_len) != NGX_OK) { - return 0; + qc->error = NGX_QUIC_ERR_INTERNAL_ERROR; } return 1; @@ -102,7 +102,7 @@ ngx_quic_set_write_secret(ngx_ssl_conn_t *ssl_conn, cipher, wsecret, secret_len) != NGX_OK) { - return 0; + qc->error = NGX_QUIC_ERR_INTERNAL_ERROR; } return 1; @@ -136,7 +136,8 @@ ngx_quic_set_encryption_secrets(ngx_ssl_conn_t *ssl_conn, cipher, rsecret, secret_len) != NGX_OK) { - return 0; + qc->error = NGX_QUIC_ERR_INTERNAL_ERROR; + return 1; } if (level == ssl_encryption_early_data) { @@ -153,7 +154,7 @@ ngx_quic_set_encryption_secrets(ngx_ssl_conn_t *ssl_conn, cipher, wsecret, secret_len) != NGX_OK) { - return 0; + qc->error = NGX_QUIC_ERR_INTERNAL_ERROR; } return 1; @@ -199,7 +200,7 @@ ngx_quic_add_handshake_data(ngx_ssl_conn_t *ssl_conn, ngx_log_error(NGX_LOG_INFO, c->log, 0, "quic unsupported protocol in ALPN extension"); - return 0; + return 1; } SSL_get_peer_quic_transport_params(ssl_conn, &client_params, @@ -216,7 +217,7 @@ ngx_quic_add_handshake_data(ngx_ssl_conn_t *ssl_conn, ngx_log_error(NGX_LOG_INFO, c->log, 0, "missing transport parameters"); - return 0; + return 1; } p = (u_char *) client_params; @@ -231,11 +232,11 @@ ngx_quic_add_handshake_data(ngx_ssl_conn_t *ssl_conn, qc->error = NGX_QUIC_ERR_TRANSPORT_PARAMETER_ERROR; qc->error_reason = "failed to process transport parameters"; - return 0; + return 1; } if (ngx_quic_apply_transport_params(c, &ctp) != NGX_OK) { - return 0; + return 1; } qc->client_tp_done = 1; @@ -245,12 +246,14 @@ ngx_quic_add_handshake_data(ngx_ssl_conn_t *ssl_conn, out = ngx_quic_copy_buffer(c, (u_char *) data, len); if (out == NGX_CHAIN_ERROR) { - return 0; + qc->error = NGX_QUIC_ERR_INTERNAL_ERROR; + return 1; } frame = ngx_quic_alloc_frame(c); if (frame == NULL) { - return 0; + qc->error = NGX_QUIC_ERR_INTERNAL_ERROR; + return 1; } frame->data = out; @@ -412,6 +415,10 @@ ngx_quic_crypto_input(ngx_connection_t *c, ngx_chain_t *data, ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, "SSL_do_handshake: %d", n); + if (qc->error != (ngx_uint_t) -1) { + return NGX_ERROR; + } + if (n <= 0) { sslerr = SSL_get_error(ssl_conn, n); From 5d7fd4a7e3025e8600bb029742a0a28bf4ca9eec Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Wed, 14 May 2025 23:33:00 +0400 Subject: [PATCH 270/279] QUIC: reset qc->error to zero again. Following the previous change that removed posting a close event in OpenSSL compat layer, now ngx_quic_close_connection() is always called on error path with either NGX_ERROR or qc->error set. This allows to remove a special value -1 served as a missing error, which simplifies the code. Partially reverts d3fb12d77. Also, this improves handling of the draining connection state, which consists of posting a close event with NGX_OK and no qc->error set, where it was previously converted to NGX_QUIC_ERR_INTERNAL_ERROR. Notably, this is rather a cosmetic fix, because drained connections do not send any packets including CONNECTION_CLOSE, and qc->error is not otherwise used. --- src/event/quic/ngx_event_quic.c | 6 +++--- src/event/quic/ngx_event_quic_ssl.c | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/event/quic/ngx_event_quic.c b/src/event/quic/ngx_event_quic.c index a4ad85d56bb..9f968d5fb6a 100644 --- a/src/event/quic/ngx_event_quic.c +++ b/src/event/quic/ngx_event_quic.c @@ -72,7 +72,7 @@ ngx_quic_connstate_dbg(ngx_connection_t *c) if (qc) { - if (qc->error != (ngx_uint_t) -1) { + if (qc->error) { p = ngx_slprintf(p, last, "%s", qc->error_app ? " app" : ""); p = ngx_slprintf(p, last, " error:%ui", qc->error); @@ -520,7 +520,7 @@ ngx_quic_close_connection(ngx_connection_t *c, ngx_int_t rc) * to terminate the connection immediately. */ - if (qc->error == (ngx_uint_t) -1) { + if (qc->error == 0 && rc == NGX_ERROR) { qc->error = NGX_QUIC_ERR_INTERNAL_ERROR; qc->error_app = 0; } @@ -961,7 +961,7 @@ ngx_quic_handle_payload(ngx_connection_t *c, ngx_quic_header_t *pkt) qc = ngx_quic_get_connection(c); - qc->error = (ngx_uint_t) -1; + qc->error = 0; qc->error_reason = 0; c->log->action = "decrypting packet"; diff --git a/src/event/quic/ngx_event_quic_ssl.c b/src/event/quic/ngx_event_quic_ssl.c index dd7ee3702fb..5b897bdb64d 100644 --- a/src/event/quic/ngx_event_quic_ssl.c +++ b/src/event/quic/ngx_event_quic_ssl.c @@ -415,7 +415,7 @@ ngx_quic_crypto_input(ngx_connection_t *c, ngx_chain_t *data, ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, "SSL_do_handshake: %d", n); - if (qc->error != (ngx_uint_t) -1) { + if (qc->error) { return NGX_ERROR; } From 54e6b7cfeeae50f708398468078094fd309828e0 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Tue, 6 May 2025 18:57:01 +0400 Subject: [PATCH 271/279] QUIC: logging missing mandatory TLS extensions only once. Previously, they might be logged on every add_handshake_data callback invocation when using OpenSSL compat layer and processing coalesced handshake messages. Further, the ALPN error message is adjusted to signal the missing extension. Possible reasons were previously narrowed down with ebb6f7d65 changes in the ALPN callback that is invoked earlier in the handshake. --- src/event/quic/ngx_event_quic_ssl.c | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/event/quic/ngx_event_quic_ssl.c b/src/event/quic/ngx_event_quic_ssl.c index 5b897bdb64d..e5d481d1ca7 100644 --- a/src/event/quic/ngx_event_quic_ssl.c +++ b/src/event/quic/ngx_event_quic_ssl.c @@ -195,11 +195,14 @@ ngx_quic_add_handshake_data(ngx_ssl_conn_t *ssl_conn, SSL_get0_alpn_selected(ssl_conn, &alpn_data, &alpn_len); if (alpn_len == 0) { - qc->error = NGX_QUIC_ERR_CRYPTO(SSL_AD_NO_APPLICATION_PROTOCOL); - qc->error_reason = "unsupported protocol in ALPN extension"; + if (qc->error == 0) { + qc->error = NGX_QUIC_ERR_CRYPTO(SSL_AD_NO_APPLICATION_PROTOCOL); + qc->error_reason = "missing ALPN extension"; + + ngx_log_error(NGX_LOG_INFO, c->log, 0, + "quic missing ALPN extension"); + } - ngx_log_error(NGX_LOG_INFO, c->log, 0, - "quic unsupported protocol in ALPN extension"); return 1; } @@ -212,11 +215,15 @@ ngx_quic_add_handshake_data(ngx_ssl_conn_t *ssl_conn, if (client_params_len == 0) { /* RFC 9001, 8.2. QUIC Transport Parameters Extension */ - qc->error = NGX_QUIC_ERR_CRYPTO(SSL_AD_MISSING_EXTENSION); - qc->error_reason = "missing transport parameters"; - ngx_log_error(NGX_LOG_INFO, c->log, 0, - "missing transport parameters"); + if (qc->error == 0) { + qc->error = NGX_QUIC_ERR_CRYPTO(SSL_AD_MISSING_EXTENSION); + qc->error_reason = "missing transport parameters"; + + ngx_log_error(NGX_LOG_INFO, c->log, 0, + "missing transport parameters"); + } + return 1; } From e561f7dbcfc27f5f648e5151de0796e691cbc1b0 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Wed, 21 May 2025 03:54:45 +0400 Subject: [PATCH 272/279] QUIC: defined SSL API macros in a single place. All definitions now set in ngx_event_quic.h, this includes moving NGX_QUIC_OPENSSL_COMPAT from autotests to compile time. Further, to improve code readability, a new NGX_QUIC_QUICTLS_API macro is used for QuicTLS that provides old BoringSSL QUIC API. --- auto/lib/openssl/conf | 3 --- src/event/quic/ngx_event_quic.c | 4 ++-- src/event/quic/ngx_event_quic.h | 12 ++++++++++++ src/event/quic/ngx_event_quic_openssl_compat.h | 8 -------- src/event/quic/ngx_event_quic_ssl.c | 13 +++---------- 5 files changed, 17 insertions(+), 23 deletions(-) diff --git a/auto/lib/openssl/conf b/auto/lib/openssl/conf index fdf430dff75..f4b00ebd672 100644 --- a/auto/lib/openssl/conf +++ b/auto/lib/openssl/conf @@ -12,7 +12,6 @@ if [ $OPENSSL != NONE ]; then if [ $USE_OPENSSL_QUIC = YES ]; then have=NGX_QUIC . auto/have - have=NGX_QUIC_OPENSSL_COMPAT . auto/have fi case "$CC" in @@ -154,8 +153,6 @@ else . auto/feature if [ $ngx_found = no ]; then - have=NGX_QUIC_OPENSSL_COMPAT . auto/have - ngx_feature="OpenSSL QUIC compatibility" ngx_feature_test="SSL_CTX_add_custom_ext(NULL, 0, 0, NULL, NULL, NULL, NULL, NULL)" diff --git a/src/event/quic/ngx_event_quic.c b/src/event/quic/ngx_event_quic.c index 9f968d5fb6a..4f2e50240e2 100644 --- a/src/event/quic/ngx_event_quic.c +++ b/src/event/quic/ngx_event_quic.c @@ -973,8 +973,8 @@ ngx_quic_handle_payload(ngx_connection_t *c, ngx_quic_header_t *pkt) return NGX_DECLINED; } -#if !defined (OPENSSL_IS_BORINGSSL) - /* OpenSSL provides read keys for an application level before it's ready */ +#if (NGX_QUIC_QUICTLS_API) + /* QuicTLS provides app read keys before completing handshake */ if (pkt->level == ssl_encryption_application && !c->ssl->handshaked) { ngx_log_error(NGX_LOG_INFO, c->log, 0, diff --git a/src/event/quic/ngx_event_quic.h b/src/event/quic/ngx_event_quic.h index 15201671d4d..50a5c214e15 100644 --- a/src/event/quic/ngx_event_quic.h +++ b/src/event/quic/ngx_event_quic.h @@ -12,6 +12,18 @@ #include +#ifdef SSL_R_MISSING_QUIC_TRANSPORT_PARAMETERS_EXTENSION +#define NGX_QUIC_QUICTLS_API 1 + +#elif (defined OPENSSL_IS_BORINGSSL || defined LIBRESSL_VERSION_NUMBER) +#define NGX_QUIC_BORINGSSL_API 1 + +#else +#define NGX_QUIC_BORINGSSL_API 1 +#define NGX_QUIC_OPENSSL_COMPAT 1 +#endif + + #define NGX_QUIC_MAX_UDP_PAYLOAD_SIZE 65527 #define NGX_QUIC_DEFAULT_ACK_DELAY_EXPONENT 3 diff --git a/src/event/quic/ngx_event_quic_openssl_compat.h b/src/event/quic/ngx_event_quic_openssl_compat.h index 77cc3cb0d83..89ee41e8981 100644 --- a/src/event/quic/ngx_event_quic_openssl_compat.h +++ b/src/event/quic/ngx_event_quic_openssl_compat.h @@ -7,11 +7,6 @@ #ifndef _NGX_EVENT_QUIC_OPENSSL_COMPAT_H_INCLUDED_ #define _NGX_EVENT_QUIC_OPENSSL_COMPAT_H_INCLUDED_ -#if defined SSL_R_MISSING_QUIC_TRANSPORT_PARAMETERS_EXTENSION \ - || defined LIBRESSL_VERSION_NUMBER -#undef NGX_QUIC_OPENSSL_COMPAT -#else - #include #include @@ -53,7 +48,4 @@ int SSL_set_quic_transport_params(SSL *ssl, const uint8_t *params, void SSL_get_peer_quic_transport_params(const SSL *ssl, const uint8_t **out_params, size_t *out_params_len); - -#endif /* TLSEXT_TYPE_quic_transport_parameters */ - #endif /* _NGX_EVENT_QUIC_OPENSSL_COMPAT_H_INCLUDED_ */ diff --git a/src/event/quic/ngx_event_quic_ssl.c b/src/event/quic/ngx_event_quic_ssl.c index e5d481d1ca7..1bb34831cbf 100644 --- a/src/event/quic/ngx_event_quic_ssl.c +++ b/src/event/quic/ngx_event_quic_ssl.c @@ -10,13 +10,6 @@ #include -#if defined OPENSSL_IS_BORINGSSL \ - || defined LIBRESSL_VERSION_NUMBER \ - || NGX_QUIC_OPENSSL_COMPAT -#define NGX_QUIC_BORINGSSL_API 1 -#endif - - /* * RFC 9000, 7.5. Cryptographic Message Buffering * @@ -32,7 +25,7 @@ static int ngx_quic_set_read_secret(ngx_ssl_conn_t *ssl_conn, static int ngx_quic_set_write_secret(ngx_ssl_conn_t *ssl_conn, enum ssl_encryption_level_t level, const SSL_CIPHER *cipher, const uint8_t *secret, size_t secret_len); -#else +#else /* NGX_QUIC_QUICTLS_API */ static int ngx_quic_set_encryption_secrets(ngx_ssl_conn_t *ssl_conn, enum ssl_encryption_level_t level, const uint8_t *read_secret, const uint8_t *write_secret, size_t secret_len); @@ -108,7 +101,7 @@ ngx_quic_set_write_secret(ngx_ssl_conn_t *ssl_conn, return 1; } -#else +#else /* NGX_QUIC_QUICTLS_API */ static int ngx_quic_set_encryption_secrets(ngx_ssl_conn_t *ssl_conn, @@ -550,7 +543,7 @@ ngx_quic_init_connection(ngx_connection_t *c) return NGX_ERROR; } -#ifdef OPENSSL_INFO_QUIC +#if (NGX_QUIC_QUICTLS_API) if (SSL_CTX_get_max_early_data(qc->conf->ssl->ctx)) { SSL_set_quic_early_data_enabled(ssl_conn, 1); } From 9857578f15352ec248813f5b3e58ca55dc82f967 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Wed, 21 May 2025 20:32:48 +0400 Subject: [PATCH 273/279] QUIC: factored out SSL_provide_quic_data() to the helper function. It is now called from ngx_quic_handle_crypto_frame(), prior to proceeding with the handshake. With this logic removed, the handshake function is renamed to ngx_quic_handshake() to better match ngx_ssl_handshake(). --- src/event/quic/ngx_event_quic_ssl.c | 53 ++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 17 deletions(-) diff --git a/src/event/quic/ngx_event_quic_ssl.c b/src/event/quic/ngx_event_quic_ssl.c index 1bb34831cbf..c9ebd70bc68 100644 --- a/src/event/quic/ngx_event_quic_ssl.c +++ b/src/event/quic/ngx_event_quic_ssl.c @@ -36,7 +36,8 @@ static int ngx_quic_add_handshake_data(ngx_ssl_conn_t *ssl_conn, static int ngx_quic_flush_flight(ngx_ssl_conn_t *ssl_conn); static int ngx_quic_send_alert(ngx_ssl_conn_t *ssl_conn, enum ssl_encryption_level_t level, uint8_t alert); -static ngx_int_t ngx_quic_crypto_input(ngx_connection_t *c, ngx_chain_t *data, +static ngx_int_t ngx_quic_handshake(ngx_connection_t *c); +static ngx_int_t ngx_quic_crypto_provide(ngx_connection_t *c, ngx_chain_t *out, enum ssl_encryption_level_t level); @@ -357,7 +358,11 @@ ngx_quic_handle_crypto_frame(ngx_connection_t *c, ngx_quic_header_t *pkt, } if (f->offset == ctx->crypto.offset) { - if (ngx_quic_crypto_input(c, frame->data, pkt->level) != NGX_OK) { + if (ngx_quic_crypto_provide(c, frame->data, pkt->level) != NGX_OK) { + return NGX_ERROR; + } + + if (ngx_quic_handshake(c) != NGX_OK) { return NGX_ERROR; } @@ -375,7 +380,11 @@ ngx_quic_handle_crypto_frame(ngx_connection_t *c, ngx_quic_header_t *pkt, cl = ngx_quic_read_buffer(c, &ctx->crypto, (uint64_t) -1); if (cl) { - if (ngx_quic_crypto_input(c, cl, pkt->level) != NGX_OK) { + if (ngx_quic_crypto_provide(c, cl, pkt->level) != NGX_OK) { + return NGX_ERROR; + } + + if (ngx_quic_handshake(c) != NGX_OK) { return NGX_ERROR; } @@ -387,12 +396,9 @@ ngx_quic_handle_crypto_frame(ngx_connection_t *c, ngx_quic_header_t *pkt, static ngx_int_t -ngx_quic_crypto_input(ngx_connection_t *c, ngx_chain_t *data, - enum ssl_encryption_level_t level) +ngx_quic_handshake(ngx_connection_t *c) { int n, sslerr; - ngx_buf_t *b; - ngx_chain_t *cl; ngx_ssl_conn_t *ssl_conn; ngx_quic_frame_t *frame; ngx_quic_connection_t *qc; @@ -401,16 +407,6 @@ ngx_quic_crypto_input(ngx_connection_t *c, ngx_chain_t *data, ssl_conn = c->ssl->connection; - for (cl = data; cl; cl = cl->next) { - b = cl->buf; - - if (!SSL_provide_quic_data(ssl_conn, level, b->pos, b->last - b->pos)) { - ngx_ssl_error(NGX_LOG_ALERT, c->log, 0, - "SSL_provide_quic_data() failed"); - return NGX_ERROR; - } - } - n = SSL_do_handshake(ssl_conn); ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, "SSL_do_handshake: %d", n); @@ -503,6 +499,29 @@ ngx_quic_crypto_input(ngx_connection_t *c, ngx_chain_t *data, } +static ngx_int_t +ngx_quic_crypto_provide(ngx_connection_t *c, ngx_chain_t *out, + enum ssl_encryption_level_t level) +{ + ngx_buf_t *b; + ngx_chain_t *cl; + + for (cl = out; cl; cl = cl->next) { + b = cl->buf; + + if (!SSL_provide_quic_data(c->ssl->connection, level, b->pos, + b->last - b->pos)) + { + ngx_ssl_error(NGX_LOG_ALERT, c->log, 0, + "SSL_provide_quic_data() failed"); + return NGX_ERROR; + } + } + + return NGX_OK; +} + + ngx_int_t ngx_quic_init_connection(ngx_connection_t *c) { From bcb9d3fd2cc88eee23a5da854a0e2aa5c5b688d7 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Tue, 6 May 2025 15:58:17 +0400 Subject: [PATCH 274/279] QUIC: ssl_encryption_level_t abstraction layer. Encryption level values are decoupled from ssl_encryption_level_t, which is now limited to BoringSSL QUIC callbacks, with mappings provided. Although the values match, this provides a technically safe approach, in particular, to access protection level sized arrays. In preparation for using OpenSSL 3.5 TLS callbacks. --- src/event/quic/ngx_event_quic.c | 36 +++---- src/event/quic/ngx_event_quic_ack.c | 16 +-- src/event/quic/ngx_event_quic_connection.h | 24 +++-- src/event/quic/ngx_event_quic_connid.c | 6 +- src/event/quic/ngx_event_quic_migration.c | 22 ++--- .../quic/ngx_event_quic_openssl_compat.c | 11 +-- src/event/quic/ngx_event_quic_output.c | 16 +-- src/event/quic/ngx_event_quic_protection.c | 19 ++-- src/event/quic/ngx_event_quic_protection.h | 14 +-- src/event/quic/ngx_event_quic_ssl.c | 98 ++++++++++++++----- src/event/quic/ngx_event_quic_streams.c | 16 +-- src/event/quic/ngx_event_quic_transport.c | 14 +-- src/event/quic/ngx_event_quic_transport.h | 10 +- 13 files changed, 174 insertions(+), 128 deletions(-) diff --git a/src/event/quic/ngx_event_quic.c b/src/event/quic/ngx_event_quic.c index 4f2e50240e2..8df4877732b 100644 --- a/src/event/quic/ngx_event_quic.c +++ b/src/event/quic/ngx_event_quic.c @@ -260,9 +260,9 @@ ngx_quic_new_connection(ngx_connection_t *c, ngx_quic_conf_t *conf, qc->send_ctx[i].pending_ack = NGX_QUIC_UNSET_PN; } - qc->send_ctx[0].level = ssl_encryption_initial; - qc->send_ctx[1].level = ssl_encryption_handshake; - qc->send_ctx[2].level = ssl_encryption_application; + qc->send_ctx[0].level = NGX_QUIC_ENCRYPTION_INITIAL; + qc->send_ctx[1].level = NGX_QUIC_ENCRYPTION_HANDSHAKE; + qc->send_ctx[2].level = NGX_QUIC_ENCRYPTION_APPLICATION; ngx_queue_init(&qc->free_frames); @@ -800,13 +800,13 @@ ngx_quic_handle_packet(ngx_connection_t *c, ngx_quic_conf_t *conf, pkt->dcid.len, &pkt->dcid); #if (NGX_DEBUG) - if (pkt->level != ssl_encryption_application) { + if (pkt->level != NGX_QUIC_ENCRYPTION_APPLICATION) { ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic packet rx scid len:%uz %xV", pkt->scid.len, &pkt->scid); } - if (pkt->level == ssl_encryption_initial) { + if (pkt->level == NGX_QUIC_ENCRYPTION_INITIAL) { ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic address validation token len:%uz %xV", pkt->token.len, &pkt->token); @@ -823,7 +823,7 @@ ngx_quic_handle_packet(ngx_connection_t *c, ngx_quic_conf_t *conf, return NGX_DECLINED; } - if (pkt->level != ssl_encryption_application) { + if (pkt->level != NGX_QUIC_ENCRYPTION_APPLICATION) { if (pkt->version != qc->version) { ngx_log_error(NGX_LOG_INFO, c->log, 0, @@ -853,7 +853,9 @@ ngx_quic_handle_packet(ngx_connection_t *c, ngx_quic_conf_t *conf, rc = ngx_quic_handle_payload(c, pkt); - if (rc == NGX_DECLINED && pkt->level == ssl_encryption_application) { + if (rc == NGX_DECLINED + && pkt->level == NGX_QUIC_ENCRYPTION_APPLICATION) + { if (ngx_quic_handle_stateless_reset(c, pkt) == NGX_OK) { ngx_log_error(NGX_LOG_INFO, c->log, 0, "quic stateless reset packet detected"); @@ -874,11 +876,11 @@ ngx_quic_handle_packet(ngx_connection_t *c, ngx_quic_conf_t *conf, return ngx_quic_negotiate_version(c, pkt); } - if (pkt->level == ssl_encryption_application) { + if (pkt->level == NGX_QUIC_ENCRYPTION_APPLICATION) { return ngx_quic_send_stateless_reset(c, conf, pkt); } - if (pkt->level != ssl_encryption_initial) { + if (pkt->level != NGX_QUIC_ENCRYPTION_INITIAL) { ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic expected initial, got handshake"); return NGX_ERROR; @@ -976,7 +978,7 @@ ngx_quic_handle_payload(ngx_connection_t *c, ngx_quic_header_t *pkt) #if (NGX_QUIC_QUICTLS_API) /* QuicTLS provides app read keys before completing handshake */ - if (pkt->level == ssl_encryption_application && !c->ssl->handshaked) { + if (pkt->level == NGX_QUIC_ENCRYPTION_APPLICATION && !c->ssl->handshaked) { ngx_log_error(NGX_LOG_INFO, c->log, 0, "quic no %s keys ready, ignoring packet", ngx_quic_level_name(pkt->level)); @@ -1014,14 +1016,14 @@ ngx_quic_handle_payload(ngx_connection_t *c, ngx_quic_header_t *pkt) } } - if (pkt->level == ssl_encryption_handshake) { + if (pkt->level == NGX_QUIC_ENCRYPTION_HANDSHAKE) { /* * RFC 9001, 4.9.1. Discarding Initial Keys * * The successful use of Handshake packets indicates * that no more Initial packets need to be exchanged */ - ngx_quic_discard_ctx(c, ssl_encryption_initial); + ngx_quic_discard_ctx(c, NGX_QUIC_ENCRYPTION_INITIAL); if (!qc->path->validated) { qc->path->validated = 1; @@ -1030,14 +1032,14 @@ ngx_quic_handle_payload(ngx_connection_t *c, ngx_quic_header_t *pkt) } } - if (pkt->level == ssl_encryption_application) { + if (pkt->level == NGX_QUIC_ENCRYPTION_APPLICATION) { /* * RFC 9001, 4.9.3. Discarding 0-RTT Keys * * After receiving a 1-RTT packet, servers MUST discard * 0-RTT keys within a short time */ - ngx_quic_keys_discard(qc->keys, ssl_encryption_early_data); + ngx_quic_keys_discard(qc->keys, NGX_QUIC_ENCRYPTION_EARLY_DATA); } if (qc->closing) { @@ -1064,7 +1066,7 @@ ngx_quic_handle_payload(ngx_connection_t *c, ngx_quic_header_t *pkt) c->log->action = "handling payload"; - if (pkt->level != ssl_encryption_application) { + if (pkt->level != NGX_QUIC_ENCRYPTION_APPLICATION) { return ngx_quic_handle_frames(c, pkt); } @@ -1089,7 +1091,7 @@ ngx_quic_handle_payload(ngx_connection_t *c, ngx_quic_header_t *pkt) void -ngx_quic_discard_ctx(ngx_connection_t *c, enum ssl_encryption_level_t level) +ngx_quic_discard_ctx(ngx_connection_t *c, ngx_uint_t level) { ngx_queue_t *q; ngx_quic_frame_t *f; @@ -1130,7 +1132,7 @@ ngx_quic_discard_ctx(ngx_connection_t *c, enum ssl_encryption_level_t level) ngx_quic_free_frame(c, f); } - if (level == ssl_encryption_initial) { + if (level == NGX_QUIC_ENCRYPTION_INITIAL) { /* close temporary listener with initial dcid */ qsock = ngx_quic_find_socket(c, NGX_QUIC_UNSET_PN); if (qsock) { diff --git a/src/event/quic/ngx_event_quic_ack.c b/src/event/quic/ngx_event_quic_ack.c index c7fd96c2cdd..abd3f7ade9a 100644 --- a/src/event/quic/ngx_event_quic_ack.c +++ b/src/event/quic/ngx_event_quic_ack.c @@ -36,7 +36,7 @@ typedef struct { static ngx_inline ngx_msec_t ngx_quic_time_threshold(ngx_quic_connection_t *qc); static uint64_t ngx_quic_packet_threshold(ngx_quic_send_ctx_t *ctx); static void ngx_quic_rtt_sample(ngx_connection_t *c, ngx_quic_ack_frame_t *ack, - enum ssl_encryption_level_t level, ngx_msec_t send_time); + ngx_uint_t level, ngx_msec_t send_time); static ngx_int_t ngx_quic_handle_ack_frame_range(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx, uint64_t min, uint64_t max, ngx_quic_ack_stat_t *st); @@ -108,7 +108,7 @@ ngx_quic_handle_ack_frame(ngx_connection_t *c, ngx_quic_header_t *pkt, ctx = ngx_quic_get_send_ctx(qc, pkt->level); ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, - "quic ngx_quic_handle_ack_frame level:%d", pkt->level); + "quic ngx_quic_handle_ack_frame level:%ui", pkt->level); ack = &f->u.ack; @@ -207,7 +207,7 @@ ngx_quic_handle_ack_frame(ngx_connection_t *c, ngx_quic_header_t *pkt, static void ngx_quic_rtt_sample(ngx_connection_t *c, ngx_quic_ack_frame_t *ack, - enum ssl_encryption_level_t level, ngx_msec_t send_time) + ngx_uint_t level, ngx_msec_t send_time) { ngx_msec_t latest_rtt, ack_delay, adjusted_rtt, rttvar_sample; ngx_quic_connection_t *qc; @@ -260,7 +260,7 @@ ngx_quic_handle_ack_frame_range(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx, qc = ngx_quic_get_connection(c); - if (ctx->level == ssl_encryption_application) { + if (ctx->level == NGX_QUIC_ENCRYPTION_APPLICATION) { if (ngx_quic_handle_path_mtu(c, qc->path, min, max) != NGX_OK) { return NGX_ERROR; } @@ -634,7 +634,7 @@ ngx_quic_detect_lost(ngx_connection_t *c, ngx_quic_ack_stat_t *st) wait = start->send_time + thr - now; ngx_log_debug5(NGX_LOG_DEBUG_EVENT, c->log, 0, - "quic detect_lost pnum:%uL thr:%M pthr:%uL wait:%i level:%d", + "quic detect_lost pnum:%uL thr:%M pthr:%uL wait:%i level:%ui", start->pnum, thr, pkt_thr, (ngx_int_t) wait, start->level); if ((ngx_msec_int_t) wait > 0 @@ -787,7 +787,7 @@ ngx_quic_resend_frames(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx) switch (f->type) { case NGX_QUIC_FT_ACK: case NGX_QUIC_FT_ACK_ECN: - if (ctx->level == ssl_encryption_application) { + if (ctx->level == NGX_QUIC_ENCRYPTION_APPLICATION) { /* force generation of most recent acknowledgment */ ctx->send_ack = NGX_QUIC_MAX_ACK_GAP; } @@ -1073,7 +1073,7 @@ ngx_quic_pto(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx) duration = qc->avg_rtt; duration += ngx_max(4 * qc->rttvar, NGX_QUIC_TIME_GRANULARITY); - if (ctx->level == ssl_encryption_application && c->ssl->handshaked) { + if (ctx->level == NGX_QUIC_ENCRYPTION_APPLICATION && c->ssl->handshaked) { duration += qc->ctp.max_ack_delay; } @@ -1428,7 +1428,7 @@ ngx_quic_generate_ack(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx) return NGX_OK; } - if (ctx->level == ssl_encryption_application) { + if (ctx->level == NGX_QUIC_ENCRYPTION_APPLICATION) { delay = ngx_current_msec - ctx->ack_delay_start; qc = ngx_quic_get_connection(c); diff --git a/src/event/quic/ngx_event_quic_connection.h b/src/event/quic/ngx_event_quic_connection.h index 04cda859ee0..856512118f9 100644 --- a/src/event/quic/ngx_event_quic_connection.h +++ b/src/event/quic/ngx_event_quic_connection.h @@ -17,6 +17,15 @@ /* #define NGX_QUIC_DEBUG_ALLOC */ /* log frames and bufs alloc */ /* #define NGX_QUIC_DEBUG_CRYPTO */ +#define NGX_QUIC_ENCRYPTION_INITIAL 0 +#define NGX_QUIC_ENCRYPTION_EARLY_DATA 1 +#define NGX_QUIC_ENCRYPTION_HANDSHAKE 2 +#define NGX_QUIC_ENCRYPTION_APPLICATION 3 +#define NGX_QUIC_ENCRYPTION_LAST 4 + +#define NGX_QUIC_SEND_CTX_LAST (NGX_QUIC_ENCRYPTION_LAST - 1) + + typedef struct ngx_quic_connection_s ngx_quic_connection_t; typedef struct ngx_quic_server_id_s ngx_quic_server_id_t; typedef struct ngx_quic_client_id_s ngx_quic_client_id_t; @@ -46,8 +55,6 @@ typedef struct ngx_quic_keys_s ngx_quic_keys_t; #define NGX_QUIC_UNSET_PN (uint64_t) -1 -#define NGX_QUIC_SEND_CTX_LAST (NGX_QUIC_ENCRYPTION_LAST - 1) - /* 0-RTT and 1-RTT data exist in the same packet number space, * so we have 3 packet number spaces: * @@ -56,9 +63,9 @@ typedef struct ngx_quic_keys_s ngx_quic_keys_t; * 2 - 0-RTT and 1-RTT */ #define ngx_quic_get_send_ctx(qc, level) \ - ((level) == ssl_encryption_initial) ? &((qc)->send_ctx[0]) \ - : (((level) == ssl_encryption_handshake) ? &((qc)->send_ctx[1]) \ - : &((qc)->send_ctx[2])) + ((level) == NGX_QUIC_ENCRYPTION_INITIAL) ? &((qc)->send_ctx[0]) \ + : (((level) == NGX_QUIC_ENCRYPTION_HANDSHAKE) ? &((qc)->send_ctx[1]) \ + : &((qc)->send_ctx[2])) #define ngx_quic_get_connection(c) \ (((c)->udp) ? (((ngx_quic_socket_t *)((c)->udp))->quic) : NULL) @@ -188,7 +195,7 @@ typedef struct { * are also Initial packets. */ struct ngx_quic_send_ctx_s { - enum ssl_encryption_level_t level; + ngx_uint_t level; ngx_quic_buffer_t crypto; uint64_t crypto_sent; @@ -279,7 +286,7 @@ struct ngx_quic_connection_s { off_t received; ngx_uint_t error; - enum ssl_encryption_level_t error_level; + ngx_uint_t error_level; ngx_uint_t error_ftype; const char *error_reason; @@ -299,8 +306,7 @@ struct ngx_quic_connection_s { ngx_int_t ngx_quic_apply_transport_params(ngx_connection_t *c, ngx_quic_tp_t *ctp); -void ngx_quic_discard_ctx(ngx_connection_t *c, - enum ssl_encryption_level_t level); +void ngx_quic_discard_ctx(ngx_connection_t *c, ngx_uint_t level); void ngx_quic_close_connection(ngx_connection_t *c, ngx_int_t rc); void ngx_quic_shutdown_quic(ngx_connection_t *c); diff --git a/src/event/quic/ngx_event_quic_connid.c b/src/event/quic/ngx_event_quic_connid.c index f5086820524..4e7b8dc2203 100644 --- a/src/event/quic/ngx_event_quic_connid.c +++ b/src/event/quic/ngx_event_quic_connid.c @@ -99,7 +99,7 @@ ngx_quic_handle_new_connection_id_frame(ngx_connection_t *c, return NGX_ERROR; } - frame->level = ssl_encryption_application; + frame->level = NGX_QUIC_ENCRYPTION_APPLICATION; frame->type = NGX_QUIC_FT_RETIRE_CONNECTION_ID; frame->u.retire_cid.sequence_number = f->seqnum; @@ -452,7 +452,7 @@ ngx_quic_send_server_id(ngx_connection_t *c, ngx_quic_server_id_t *sid) return NGX_ERROR; } - frame->level = ssl_encryption_application; + frame->level = NGX_QUIC_ENCRYPTION_APPLICATION; frame->type = NGX_QUIC_FT_NEW_CONNECTION_ID; frame->u.ncid.seqnum = sid->seqnum; frame->u.ncid.retire = 0; @@ -485,7 +485,7 @@ ngx_quic_free_client_id(ngx_connection_t *c, ngx_quic_client_id_t *cid) return NGX_ERROR; } - frame->level = ssl_encryption_application; + frame->level = NGX_QUIC_ENCRYPTION_APPLICATION; frame->type = NGX_QUIC_FT_RETIRE_CONNECTION_ID; frame->u.retire_cid.sequence_number = cid->seqnum; diff --git a/src/event/quic/ngx_event_quic_migration.c b/src/event/quic/ngx_event_quic_migration.c index 6befc34270b..42354ca6697 100644 --- a/src/event/quic/ngx_event_quic_migration.c +++ b/src/event/quic/ngx_event_quic_migration.c @@ -40,7 +40,7 @@ ngx_quic_handle_path_challenge_frame(ngx_connection_t *c, ngx_quic_frame_t *fp; ngx_quic_connection_t *qc; - if (pkt->level != ssl_encryption_application || pkt->path_challenged) { + if (pkt->level != NGX_QUIC_ENCRYPTION_APPLICATION || pkt->path_challenged) { ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic ignoring PATH_CHALLENGE"); return NGX_OK; @@ -55,7 +55,7 @@ ngx_quic_handle_path_challenge_frame(ngx_connection_t *c, return NGX_ERROR; } - fp->level = ssl_encryption_application; + fp->level = NGX_QUIC_ENCRYPTION_APPLICATION; fp->type = NGX_QUIC_FT_PATH_RESPONSE; fp->u.path_response = *f; @@ -93,7 +93,7 @@ ngx_quic_handle_path_challenge_frame(ngx_connection_t *c, return NGX_ERROR; } - fp->level = ssl_encryption_application; + fp->level = NGX_QUIC_ENCRYPTION_APPLICATION; fp->type = NGX_QUIC_FT_PING; ngx_quic_queue_frame(qc, fp); @@ -177,7 +177,7 @@ ngx_quic_handle_path_response_frame(ngx_connection_t *c, if (rst) { /* prevent old path packets contribution to congestion control */ - ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_application); + ctx = ngx_quic_get_send_ctx(qc, NGX_QUIC_ENCRYPTION_APPLICATION); qc->rst_pnum = ctx->pnum; ngx_memzero(&qc->congestion, sizeof(ngx_quic_congestion_t)); @@ -549,7 +549,7 @@ ngx_quic_validate_path(ngx_connection_t *c, ngx_quic_path_t *path) (void) ngx_quic_send_path_challenge(c, path); - ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_application); + ctx = ngx_quic_get_send_ctx(qc, NGX_QUIC_ENCRYPTION_APPLICATION); pto = ngx_max(ngx_quic_pto(c, ctx), 1000); path->expires = ngx_current_msec + pto; @@ -579,7 +579,7 @@ ngx_quic_send_path_challenge(ngx_connection_t *c, ngx_quic_path_t *path) return NGX_ERROR; } - frame->level = ssl_encryption_application; + frame->level = NGX_QUIC_ENCRYPTION_APPLICATION; frame->type = NGX_QUIC_FT_PATH_CHALLENGE; ngx_memcpy(frame->u.path_challenge.data, path->challenge[n], 8); @@ -767,7 +767,7 @@ ngx_quic_expire_path_validation(ngx_connection_t *c, ngx_quic_path_t *path) ngx_quic_connection_t *qc; qc = ngx_quic_get_connection(c); - ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_application); + ctx = ngx_quic_get_send_ctx(qc, NGX_QUIC_ENCRYPTION_APPLICATION); if (++path->tries < NGX_QUIC_PATH_RETRIES) { pto = ngx_max(ngx_quic_pto(c, ctx), 1000) << path->tries; @@ -830,7 +830,7 @@ ngx_quic_expire_path_mtu_delay(ngx_connection_t *c, ngx_quic_path_t *path) ngx_quic_connection_t *qc; qc = ngx_quic_get_connection(c); - ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_application); + ctx = ngx_quic_get_send_ctx(qc, NGX_QUIC_ENCRYPTION_APPLICATION); path->tries = 0; @@ -876,7 +876,7 @@ ngx_quic_expire_path_mtu_discovery(ngx_connection_t *c, ngx_quic_path_t *path) ngx_quic_connection_t *qc; qc = ngx_quic_get_connection(c); - ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_application); + ctx = ngx_quic_get_send_ctx(qc, NGX_QUIC_ENCRYPTION_APPLICATION); if (++path->tries < NGX_QUIC_PATH_RETRIES) { rc = ngx_quic_send_path_mtu_probe(c, path); @@ -922,13 +922,13 @@ ngx_quic_send_path_mtu_probe(ngx_connection_t *c, ngx_quic_path_t *path) return NGX_ERROR; } - frame->level = ssl_encryption_application; + frame->level = NGX_QUIC_ENCRYPTION_APPLICATION; frame->type = NGX_QUIC_FT_PING; frame->ignore_loss = 1; frame->ignore_congestion = 1; qc = ngx_quic_get_connection(c); - ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_application); + ctx = ngx_quic_get_send_ctx(qc, NGX_QUIC_ENCRYPTION_APPLICATION); pnum = ctx->pnum; ngx_log_debug4(NGX_LOG_DEBUG_EVENT, c->log, 0, diff --git a/src/event/quic/ngx_event_quic_openssl_compat.c b/src/event/quic/ngx_event_quic_openssl_compat.c index c5762f15521..58298dcb816 100644 --- a/src/event/quic/ngx_event_quic_openssl_compat.c +++ b/src/event/quic/ngx_event_quic_openssl_compat.c @@ -433,8 +433,7 @@ ngx_quic_compat_message_callback(int write_p, int version, int content_type, case SSL3_RT_HANDSHAKE: ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, - "quic compat tx %s len:%uz ", - ngx_quic_level_name(level), len); + "quic compat tx level:%d len:%uz", level, len); if (com->method->add_handshake_data(ssl, level, buf, len) != 1) { return; @@ -447,8 +446,8 @@ ngx_quic_compat_message_callback(int write_p, int version, int content_type, alert = ((u_char *) buf)[1]; ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0, - "quic compat %s alert:%ui len:%uz ", - ngx_quic_level_name(level), alert, len); + "quic compat level:%d alert:%ui len:%uz", + level, alert, len); if (com->method->send_alert(ssl, level, alert) != 1) { return; @@ -481,8 +480,8 @@ SSL_provide_quic_data(SSL *ssl, enum ssl_encryption_level_t level, c = ngx_ssl_get_connection(ssl); - ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic compat rx %s len:%uz", - ngx_quic_level_name(level), len); + ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic compat rx level:%d len:%uz", level, len); qc = ngx_quic_get_connection(c); com = qc->compat; diff --git a/src/event/quic/ngx_event_quic_output.c b/src/event/quic/ngx_event_quic_output.c index 01f1f911324..8c33505048b 100644 --- a/src/event/quic/ngx_event_quic_output.c +++ b/src/event/quic/ngx_event_quic_output.c @@ -294,17 +294,17 @@ ngx_quic_allow_segmentation(ngx_connection_t *c) return 0; } - ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_initial); + ctx = ngx_quic_get_send_ctx(qc, NGX_QUIC_ENCRYPTION_INITIAL); if (!ngx_queue_empty(&ctx->frames)) { return 0; } - ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_handshake); + ctx = ngx_quic_get_send_ctx(qc, NGX_QUIC_ENCRYPTION_HANDSHAKE); if (!ngx_queue_empty(&ctx->frames)) { return 0; } - ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_application); + ctx = ngx_quic_get_send_ctx(qc, NGX_QUIC_ENCRYPTION_APPLICATION); bytes = 0; len = ngx_min(qc->path->mtu, NGX_QUIC_MAX_UDP_SEGMENT_BUF); @@ -349,7 +349,7 @@ ngx_quic_create_segments(ngx_connection_t *c) cg = &qc->congestion; path = qc->path; - ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_application); + ctx = ngx_quic_get_send_ctx(qc, NGX_QUIC_ENCRYPTION_APPLICATION); if (ngx_quic_generate_ack(c, ctx) != NGX_OK) { return NGX_ERROR; @@ -500,7 +500,7 @@ ngx_quic_get_padding_level(ngx_connection_t *c) */ qc = ngx_quic_get_connection(c); - ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_initial); + ctx = ngx_quic_get_send_ctx(qc, NGX_QUIC_ENCRYPTION_INITIAL); for (q = ngx_queue_head(&ctx->frames); q != ngx_queue_sentinel(&ctx->frames); @@ -687,10 +687,10 @@ ngx_quic_init_packet(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx, pkt->flags = NGX_QUIC_PKT_FIXED_BIT; - if (ctx->level == ssl_encryption_initial) { + if (ctx->level == NGX_QUIC_ENCRYPTION_INITIAL) { pkt->flags |= NGX_QUIC_PKT_LONG | NGX_QUIC_PKT_INITIAL; - } else if (ctx->level == ssl_encryption_handshake) { + } else if (ctx->level == NGX_QUIC_ENCRYPTION_HANDSHAKE) { pkt->flags |= NGX_QUIC_PKT_LONG | NGX_QUIC_PKT_HANDSHAKE; } else { @@ -1103,7 +1103,7 @@ ngx_quic_send_new_token(ngx_connection_t *c, ngx_quic_path_t *path) return NGX_ERROR; } - frame->level = ssl_encryption_application; + frame->level = NGX_QUIC_ENCRYPTION_APPLICATION; frame->type = NGX_QUIC_FT_NEW_TOKEN; frame->data = out; frame->u.token.length = token.len; diff --git a/src/event/quic/ngx_event_quic_protection.c b/src/event/quic/ngx_event_quic_protection.c index e5c0df7b442..885843d72d8 100644 --- a/src/event/quic/ngx_event_quic_protection.c +++ b/src/event/quic/ngx_event_quic_protection.c @@ -130,8 +130,8 @@ ngx_quic_keys_set_initial_secret(ngx_quic_keys_t *keys, ngx_str_t *secret, 0x9a, 0xe6, 0xa4, 0xc8, 0x0c, 0xad, 0xcc, 0xbb, 0x7f, 0x0a }; - client = &keys->secrets[ssl_encryption_initial].client; - server = &keys->secrets[ssl_encryption_initial].server; + client = &keys->secrets[NGX_QUIC_ENCRYPTION_INITIAL].client; + server = &keys->secrets[NGX_QUIC_ENCRYPTION_INITIAL].server; /* * RFC 9001, section 5. Packet Protection @@ -656,8 +656,8 @@ ngx_quic_crypto_hp_cleanup(ngx_quic_secret_t *s) ngx_int_t ngx_quic_keys_set_encryption_secret(ngx_log_t *log, ngx_uint_t is_write, - ngx_quic_keys_t *keys, enum ssl_encryption_level_t level, - const SSL_CIPHER *cipher, const uint8_t *secret, size_t secret_len) + ngx_quic_keys_t *keys, ngx_uint_t level, const SSL_CIPHER *cipher, + const uint8_t *secret, size_t secret_len) { ngx_int_t key_len; ngx_str_t secret_str; @@ -722,8 +722,8 @@ ngx_quic_keys_set_encryption_secret(ngx_log_t *log, ngx_uint_t is_write, ngx_uint_t -ngx_quic_keys_available(ngx_quic_keys_t *keys, - enum ssl_encryption_level_t level, ngx_uint_t is_write) +ngx_quic_keys_available(ngx_quic_keys_t *keys, ngx_uint_t level, + ngx_uint_t is_write) { if (is_write == 0) { return keys->secrets[level].client.ctx != NULL; @@ -734,8 +734,7 @@ ngx_quic_keys_available(ngx_quic_keys_t *keys, void -ngx_quic_keys_discard(ngx_quic_keys_t *keys, - enum ssl_encryption_level_t level) +ngx_quic_keys_discard(ngx_quic_keys_t *keys, ngx_uint_t level) { ngx_quic_secret_t *client, *server; @@ -765,7 +764,7 @@ ngx_quic_keys_switch(ngx_connection_t *c, ngx_quic_keys_t *keys) { ngx_quic_secrets_t *current, *next, tmp; - current = &keys->secrets[ssl_encryption_application]; + current = &keys->secrets[NGX_QUIC_ENCRYPTION_APPLICATION]; next = &keys->next_key; ngx_quic_crypto_cleanup(¤t->client); @@ -794,7 +793,7 @@ ngx_quic_keys_update(ngx_event_t *ev) qc = ngx_quic_get_connection(c); keys = qc->keys; - current = &keys->secrets[ssl_encryption_application]; + current = &keys->secrets[NGX_QUIC_ENCRYPTION_APPLICATION]; next = &keys->next_key; ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic key update"); diff --git a/src/event/quic/ngx_event_quic_protection.h b/src/event/quic/ngx_event_quic_protection.h index c09456f53cd..fddc6083a32 100644 --- a/src/event/quic/ngx_event_quic_protection.h +++ b/src/event/quic/ngx_event_quic_protection.h @@ -14,8 +14,6 @@ #include -#define NGX_QUIC_ENCRYPTION_LAST ((ssl_encryption_application) + 1) - /* RFC 5116, 5.1/5.3 and RFC 8439, 2.3/2.5 for all supported ciphers */ #define NGX_QUIC_IV_LEN 12 #define NGX_QUIC_TAG_LEN 16 @@ -94,13 +92,11 @@ typedef struct { ngx_int_t ngx_quic_keys_set_initial_secret(ngx_quic_keys_t *keys, ngx_str_t *secret, ngx_log_t *log); ngx_int_t ngx_quic_keys_set_encryption_secret(ngx_log_t *log, - ngx_uint_t is_write, ngx_quic_keys_t *keys, - enum ssl_encryption_level_t level, const SSL_CIPHER *cipher, - const uint8_t *secret, size_t secret_len); -ngx_uint_t ngx_quic_keys_available(ngx_quic_keys_t *keys, - enum ssl_encryption_level_t level, ngx_uint_t is_write); -void ngx_quic_keys_discard(ngx_quic_keys_t *keys, - enum ssl_encryption_level_t level); + ngx_uint_t is_write, ngx_quic_keys_t *keys, ngx_uint_t level, + const SSL_CIPHER *cipher, const uint8_t *secret, size_t secret_len); +ngx_uint_t ngx_quic_keys_available(ngx_quic_keys_t *keys, ngx_uint_t level, + ngx_uint_t is_write); +void ngx_quic_keys_discard(ngx_quic_keys_t *keys, ngx_uint_t level); void ngx_quic_keys_switch(ngx_connection_t *c, ngx_quic_keys_t *keys); void ngx_quic_keys_update(ngx_event_t *ev); void ngx_quic_keys_cleanup(ngx_quic_keys_t *keys); diff --git a/src/event/quic/ngx_event_quic_ssl.c b/src/event/quic/ngx_event_quic_ssl.c index c9ebd70bc68..fc8ebd8cf93 100644 --- a/src/event/quic/ngx_event_quic_ssl.c +++ b/src/event/quic/ngx_event_quic_ssl.c @@ -18,44 +18,65 @@ #define NGX_QUIC_MAX_BUFFERED 65535 +static ngx_inline ngx_uint_t ngx_quic_map_encryption_level( + enum ssl_encryption_level_t ssl_level); + #if (NGX_QUIC_BORINGSSL_API) static int ngx_quic_set_read_secret(ngx_ssl_conn_t *ssl_conn, - enum ssl_encryption_level_t level, const SSL_CIPHER *cipher, + enum ssl_encryption_level_t ssl_level, const SSL_CIPHER *cipher, const uint8_t *secret, size_t secret_len); static int ngx_quic_set_write_secret(ngx_ssl_conn_t *ssl_conn, - enum ssl_encryption_level_t level, const SSL_CIPHER *cipher, + enum ssl_encryption_level_t ssl_level, const SSL_CIPHER *cipher, const uint8_t *secret, size_t secret_len); #else /* NGX_QUIC_QUICTLS_API */ static int ngx_quic_set_encryption_secrets(ngx_ssl_conn_t *ssl_conn, - enum ssl_encryption_level_t level, const uint8_t *read_secret, + enum ssl_encryption_level_t ssl_level, const uint8_t *read_secret, const uint8_t *write_secret, size_t secret_len); #endif static int ngx_quic_add_handshake_data(ngx_ssl_conn_t *ssl_conn, - enum ssl_encryption_level_t level, const uint8_t *data, size_t len); + enum ssl_encryption_level_t ssl_level, const uint8_t *data, size_t len); static int ngx_quic_flush_flight(ngx_ssl_conn_t *ssl_conn); static int ngx_quic_send_alert(ngx_ssl_conn_t *ssl_conn, - enum ssl_encryption_level_t level, uint8_t alert); + enum ssl_encryption_level_t ssl_level, uint8_t alert); static ngx_int_t ngx_quic_handshake(ngx_connection_t *c); static ngx_int_t ngx_quic_crypto_provide(ngx_connection_t *c, ngx_chain_t *out, - enum ssl_encryption_level_t level); + ngx_uint_t level); + + +static ngx_inline ngx_uint_t +ngx_quic_map_encryption_level(enum ssl_encryption_level_t ssl_level) +{ + switch (ssl_level) { + case ssl_encryption_initial: + return NGX_QUIC_ENCRYPTION_INITIAL; + case ssl_encryption_early_data: + return NGX_QUIC_ENCRYPTION_EARLY_DATA; + case ssl_encryption_handshake: + return NGX_QUIC_ENCRYPTION_HANDSHAKE; + default: /* ssl_encryption_application */ + return NGX_QUIC_ENCRYPTION_APPLICATION; + } +} #if (NGX_QUIC_BORINGSSL_API) static int ngx_quic_set_read_secret(ngx_ssl_conn_t *ssl_conn, - enum ssl_encryption_level_t level, const SSL_CIPHER *cipher, + enum ssl_encryption_level_t ssl_level, const SSL_CIPHER *cipher, const uint8_t *rsecret, size_t secret_len) { + ngx_uint_t level; ngx_connection_t *c; ngx_quic_connection_t *qc; c = ngx_ssl_get_connection(ssl_conn); qc = ngx_quic_get_connection(c); + level = ngx_quic_map_encryption_level(ssl_level); ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, - "quic ngx_quic_set_read_secret() level:%d", level); + "quic ngx_quic_set_read_secret() level:%d", ssl_level); #ifdef NGX_QUIC_DEBUG_CRYPTO ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic read secret len:%uz %*xs", secret_len, @@ -75,17 +96,19 @@ ngx_quic_set_read_secret(ngx_ssl_conn_t *ssl_conn, static int ngx_quic_set_write_secret(ngx_ssl_conn_t *ssl_conn, - enum ssl_encryption_level_t level, const SSL_CIPHER *cipher, + enum ssl_encryption_level_t ssl_level, const SSL_CIPHER *cipher, const uint8_t *wsecret, size_t secret_len) { + ngx_uint_t level; ngx_connection_t *c; ngx_quic_connection_t *qc; c = ngx_ssl_get_connection(ssl_conn); qc = ngx_quic_get_connection(c); + level = ngx_quic_map_encryption_level(ssl_level); ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, - "quic ngx_quic_set_write_secret() level:%d", level); + "quic ngx_quic_set_write_secret() level:%d", ssl_level); #ifdef NGX_QUIC_DEBUG_CRYPTO ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic write secret len:%uz %*xs", secret_len, @@ -106,18 +129,21 @@ ngx_quic_set_write_secret(ngx_ssl_conn_t *ssl_conn, static int ngx_quic_set_encryption_secrets(ngx_ssl_conn_t *ssl_conn, - enum ssl_encryption_level_t level, const uint8_t *rsecret, + enum ssl_encryption_level_t ssl_level, const uint8_t *rsecret, const uint8_t *wsecret, size_t secret_len) { + ngx_uint_t level; ngx_connection_t *c; const SSL_CIPHER *cipher; ngx_quic_connection_t *qc; c = ngx_ssl_get_connection(ssl_conn); qc = ngx_quic_get_connection(c); + level = ngx_quic_map_encryption_level(ssl_level); ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, - "quic ngx_quic_set_encryption_secrets() level:%d", level); + "quic ngx_quic_set_encryption_secrets() level:%d", + ssl_level); #ifdef NGX_QUIC_DEBUG_CRYPTO ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic read secret len:%uz %*xs", secret_len, @@ -134,7 +160,7 @@ ngx_quic_set_encryption_secrets(ngx_ssl_conn_t *ssl_conn, return 1; } - if (level == ssl_encryption_early_data) { + if (level == NGX_QUIC_ENCRYPTION_EARLY_DATA) { return 1; } @@ -159,10 +185,11 @@ ngx_quic_set_encryption_secrets(ngx_ssl_conn_t *ssl_conn, static int ngx_quic_add_handshake_data(ngx_ssl_conn_t *ssl_conn, - enum ssl_encryption_level_t level, const uint8_t *data, size_t len) + enum ssl_encryption_level_t ssl_level, const uint8_t *data, size_t len) { u_char *p, *end; size_t client_params_len; + ngx_uint_t level; ngx_chain_t *out; unsigned int alpn_len; const uint8_t *client_params; @@ -175,6 +202,7 @@ ngx_quic_add_handshake_data(ngx_ssl_conn_t *ssl_conn, c = ngx_ssl_get_connection(ssl_conn); qc = ngx_quic_get_connection(c); + level = ngx_quic_map_encryption_level(ssl_level); ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic ngx_quic_add_handshake_data"); @@ -287,8 +315,8 @@ ngx_quic_flush_flight(ngx_ssl_conn_t *ssl_conn) static int -ngx_quic_send_alert(ngx_ssl_conn_t *ssl_conn, enum ssl_encryption_level_t level, - uint8_t alert) +ngx_quic_send_alert(ngx_ssl_conn_t *ssl_conn, + enum ssl_encryption_level_t ssl_level, uint8_t alert) { ngx_connection_t *c; ngx_quic_connection_t *qc; @@ -296,8 +324,8 @@ ngx_quic_send_alert(ngx_ssl_conn_t *ssl_conn, enum ssl_encryption_level_t level, c = ngx_ssl_get_connection(ssl_conn); ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, - "quic ngx_quic_send_alert() level:%s alert:%d", - ngx_quic_level_name(level), (int) alert); + "quic ngx_quic_send_alert() level:%d alert:%d", + ssl_level, (int) alert); /* already closed on regular shutdown */ @@ -341,13 +369,13 @@ ngx_quic_handle_crypto_frame(ngx_connection_t *c, ngx_quic_header_t *pkt, } if (last <= ctx->crypto.offset) { - if (pkt->level == ssl_encryption_initial) { + if (pkt->level == NGX_QUIC_ENCRYPTION_INITIAL) { /* speeding up handshake completion */ if (!ngx_queue_empty(&ctx->sent)) { ngx_quic_resend_frames(c, ctx); - ctx = ngx_quic_get_send_ctx(qc, ssl_encryption_handshake); + ctx = ngx_quic_get_send_ctx(qc, NGX_QUIC_ENCRYPTION_HANDSHAKE); while (!ngx_queue_empty(&ctx->sent)) { ngx_quic_resend_frames(c, ctx); } @@ -436,7 +464,7 @@ ngx_quic_handshake(ngx_connection_t *c) } if (n <= 0 || SSL_in_init(ssl_conn)) { - if (ngx_quic_keys_available(qc->keys, ssl_encryption_early_data, 0) + if (ngx_quic_keys_available(qc->keys, NGX_QUIC_ENCRYPTION_EARLY_DATA, 0) && qc->client_tp_done) { if (ngx_quic_init_streams(c) != NGX_OK) { @@ -458,7 +486,7 @@ ngx_quic_handshake(ngx_connection_t *c) return NGX_ERROR; } - frame->level = ssl_encryption_application; + frame->level = NGX_QUIC_ENCRYPTION_APPLICATION; frame->type = NGX_QUIC_FT_HANDSHAKE_DONE; ngx_quic_queue_frame(qc, frame); @@ -482,7 +510,7 @@ ngx_quic_handshake(ngx_connection_t *c) * An endpoint MUST discard its Handshake keys * when the TLS handshake is confirmed. */ - ngx_quic_discard_ctx(c, ssl_encryption_handshake); + ngx_quic_discard_ctx(c, NGX_QUIC_ENCRYPTION_HANDSHAKE); ngx_quic_discover_path_mtu(c, qc->path); @@ -501,15 +529,31 @@ ngx_quic_handshake(ngx_connection_t *c) static ngx_int_t ngx_quic_crypto_provide(ngx_connection_t *c, ngx_chain_t *out, - enum ssl_encryption_level_t level) + ngx_uint_t level) { - ngx_buf_t *b; - ngx_chain_t *cl; + ngx_buf_t *b; + ngx_chain_t *cl; + enum ssl_encryption_level_t ssl_level; + + switch (level) { + case NGX_QUIC_ENCRYPTION_INITIAL: + ssl_level = ssl_encryption_initial; + break; + case NGX_QUIC_ENCRYPTION_EARLY_DATA: + ssl_level = ssl_encryption_early_data; + break; + case NGX_QUIC_ENCRYPTION_HANDSHAKE: + ssl_level = ssl_encryption_handshake; + break; + default: /* NGX_QUIC_ENCRYPTION_APPLICATION */ + ssl_level = ssl_encryption_application; + break; + } for (cl = out; cl; cl = cl->next) { b = cl->buf; - if (!SSL_provide_quic_data(c->ssl->connection, level, b->pos, + if (!SSL_provide_quic_data(c->ssl->connection, ssl_level, b->pos, b->last - b->pos)) { ngx_ssl_error(NGX_LOG_ALERT, c->log, 0, diff --git a/src/event/quic/ngx_event_quic_streams.c b/src/event/quic/ngx_event_quic_streams.c index a9a21f5785f..18fffeabe4f 100644 --- a/src/event/quic/ngx_event_quic_streams.c +++ b/src/event/quic/ngx_event_quic_streams.c @@ -280,7 +280,7 @@ ngx_quic_do_reset_stream(ngx_quic_stream_t *qs, ngx_uint_t err) return NGX_ERROR; } - frame->level = ssl_encryption_application; + frame->level = NGX_QUIC_ENCRYPTION_APPLICATION; frame->type = NGX_QUIC_FT_RESET_STREAM; frame->u.reset_stream.id = qs->id; frame->u.reset_stream.error_code = err; @@ -367,7 +367,7 @@ ngx_quic_shutdown_stream_recv(ngx_connection_t *c) ngx_log_debug1(NGX_LOG_DEBUG_EVENT, pc->log, 0, "quic stream id:0x%xL recv shutdown", qs->id); - frame->level = ssl_encryption_application; + frame->level = NGX_QUIC_ENCRYPTION_APPLICATION; frame->type = NGX_QUIC_FT_STOP_SENDING; frame->u.stop_sending.id = qs->id; frame->u.stop_sending.error_code = qc->conf->stream_close_code; @@ -527,7 +527,7 @@ ngx_quic_reject_stream(ngx_connection_t *c, uint64_t id) return NGX_ERROR; } - frame->level = ssl_encryption_application; + frame->level = NGX_QUIC_ENCRYPTION_APPLICATION; frame->type = NGX_QUIC_FT_RESET_STREAM; frame->u.reset_stream.id = id; frame->u.reset_stream.error_code = code; @@ -540,7 +540,7 @@ ngx_quic_reject_stream(ngx_connection_t *c, uint64_t id) return NGX_ERROR; } - frame->level = ssl_encryption_application; + frame->level = NGX_QUIC_ENCRYPTION_APPLICATION; frame->type = NGX_QUIC_FT_STOP_SENDING; frame->u.stop_sending.id = id; frame->u.stop_sending.error_code = code; @@ -1062,7 +1062,7 @@ ngx_quic_stream_flush(ngx_quic_stream_t *qs) return NGX_ERROR; } - frame->level = ssl_encryption_application; + frame->level = NGX_QUIC_ENCRYPTION_APPLICATION; frame->type = NGX_QUIC_FT_STREAM; frame->data = out; @@ -1180,7 +1180,7 @@ ngx_quic_close_stream(ngx_quic_stream_t *qs) return NGX_ERROR; } - frame->level = ssl_encryption_application; + frame->level = NGX_QUIC_ENCRYPTION_APPLICATION; frame->type = NGX_QUIC_FT_MAX_STREAMS; if (qs->id & NGX_QUIC_STREAM_UNIDIRECTIONAL) { @@ -1771,7 +1771,7 @@ ngx_quic_update_max_stream_data(ngx_quic_stream_t *qs) return NGX_ERROR; } - frame->level = ssl_encryption_application; + frame->level = NGX_QUIC_ENCRYPTION_APPLICATION; frame->type = NGX_QUIC_FT_MAX_STREAM_DATA; frame->u.max_stream_data.id = qs->id; frame->u.max_stream_data.limit = qs->recv_max_data; @@ -1807,7 +1807,7 @@ ngx_quic_update_max_data(ngx_connection_t *c) return NGX_ERROR; } - frame->level = ssl_encryption_application; + frame->level = NGX_QUIC_ENCRYPTION_APPLICATION; frame->type = NGX_QUIC_FT_MAX_DATA; frame->u.max_data.max_data = qc->streams.recv_max_data; diff --git a/src/event/quic/ngx_event_quic_transport.c b/src/event/quic/ngx_event_quic_transport.c index 0b3ef4b2ed4..ba6211c33d4 100644 --- a/src/event/quic/ngx_event_quic_transport.c +++ b/src/event/quic/ngx_event_quic_transport.c @@ -281,7 +281,7 @@ ngx_int_t ngx_quic_parse_packet(ngx_quic_header_t *pkt) { if (!ngx_quic_long_pkt(pkt->flags)) { - pkt->level = ssl_encryption_application; + pkt->level = NGX_QUIC_ENCRYPTION_APPLICATION; if (ngx_quic_parse_short_header(pkt, NGX_QUIC_SERVER_CID_LEN) != NGX_OK) { @@ -468,13 +468,13 @@ ngx_quic_parse_long_header_v1(ngx_quic_header_t *pkt) return NGX_ERROR; } - pkt->level = ssl_encryption_initial; + pkt->level = NGX_QUIC_ENCRYPTION_INITIAL; } else if (ngx_quic_pkt_zrtt(pkt->flags)) { - pkt->level = ssl_encryption_early_data; + pkt->level = NGX_QUIC_ENCRYPTION_EARLY_DATA; } else if (ngx_quic_pkt_hs(pkt->flags)) { - pkt->level = ssl_encryption_handshake; + pkt->level = NGX_QUIC_ENCRYPTION_HANDSHAKE; } else { ngx_log_error(NGX_LOG_INFO, pkt->log, 0, @@ -593,7 +593,7 @@ ngx_quic_payload_size(ngx_quic_header_t *pkt, size_t pkt_len) /* flags, version, dcid and scid with lengths and zero-length token */ len = 5 + 2 + pkt->dcid.len + pkt->scid.len - + (pkt->level == ssl_encryption_initial ? 1 : 0); + + (pkt->level == NGX_QUIC_ENCRYPTION_INITIAL ? 1 : 0); if (len > pkt_len) { return 0; @@ -632,7 +632,7 @@ ngx_quic_create_long_header(ngx_quic_header_t *pkt, u_char *out, if (out == NULL) { return 5 + 2 + pkt->dcid.len + pkt->scid.len + ngx_quic_varint_len(rem_len) + pkt->num_len - + (pkt->level == ssl_encryption_initial ? 1 : 0); + + (pkt->level == NGX_QUIC_ENCRYPTION_INITIAL ? 1 : 0); } p = start = out; @@ -647,7 +647,7 @@ ngx_quic_create_long_header(ngx_quic_header_t *pkt, u_char *out, *p++ = pkt->scid.len; p = ngx_cpymem(p, pkt->scid.data, pkt->scid.len); - if (pkt->level == ssl_encryption_initial) { + if (pkt->level == NGX_QUIC_ENCRYPTION_INITIAL) { ngx_quic_build_int(&p, 0); } diff --git a/src/event/quic/ngx_event_quic_transport.h b/src/event/quic/ngx_event_quic_transport.h index dcd763df1ff..656cb09fb55 100644 --- a/src/event/quic/ngx_event_quic_transport.h +++ b/src/event/quic/ngx_event_quic_transport.h @@ -47,9 +47,9 @@ (ngx_quic_long_pkt(flags) ? 0x0F : 0x1F) #define ngx_quic_level_name(lvl) \ - (lvl == ssl_encryption_application) ? "app" \ - : (lvl == ssl_encryption_initial) ? "init" \ - : (lvl == ssl_encryption_handshake) ? "hs" : "early" + (lvl == NGX_QUIC_ENCRYPTION_APPLICATION) ? "app" \ + : (lvl == NGX_QUIC_ENCRYPTION_INITIAL) ? "init" \ + : (lvl == NGX_QUIC_ENCRYPTION_HANDSHAKE) ? "hs" : "early" #define NGX_QUIC_MAX_CID_LEN 20 #define NGX_QUIC_SERVER_CID_LEN NGX_QUIC_MAX_CID_LEN @@ -262,7 +262,7 @@ typedef struct ngx_quic_frame_s ngx_quic_frame_t; struct ngx_quic_frame_s { ngx_uint_t type; - enum ssl_encryption_level_t level; + ngx_uint_t level; ngx_queue_t queue; uint64_t pnum; size_t plen; @@ -310,7 +310,7 @@ typedef struct { uint8_t flags; uint32_t version; ngx_str_t token; - enum ssl_encryption_level_t level; + ngx_uint_t level; ngx_uint_t error; /* filled in by parser */ From 1d4d2f2c962c33aafdd8f79d9fc50b7cacf05e24 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Fri, 16 May 2025 01:10:11 +0400 Subject: [PATCH 275/279] QUIC: better approach for premature handshake completion. Using SSL_in_init() to inspect a handshake state was replaced with SSL_is_init_finished(). This represents a more complete fix to the BoringSSL issue addressed in 22671b37e. This provides awareness of the early data handshake state when using OpenSSL 3.5 TLS callbacks in 0-RTT enabled configurations, which, in particular, is used to avoid premature completion of the initial TLS handshake, before required client handshake messages are received. This is a non-functional change when using BoringSSL. It supersedes testing non-positive SSL_do_handshake() results in all supported SSL libraries, hence simplified. In preparation for using OpenSSL 3.5 TLS callbacks. --- src/event/quic/ngx_event_quic_ssl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/event/quic/ngx_event_quic_ssl.c b/src/event/quic/ngx_event_quic_ssl.c index fc8ebd8cf93..6ce926c8187 100644 --- a/src/event/quic/ngx_event_quic_ssl.c +++ b/src/event/quic/ngx_event_quic_ssl.c @@ -463,7 +463,7 @@ ngx_quic_handshake(ngx_connection_t *c) } } - if (n <= 0 || SSL_in_init(ssl_conn)) { + if (!SSL_is_init_finished(ssl_conn)) { if (ngx_quic_keys_available(qc->keys, NGX_QUIC_ENCRYPTION_EARLY_DATA, 0) && qc->client_tp_done) { From 6a134dfd4888fc3850d22294687cfb3940994c69 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Thu, 13 Feb 2025 17:00:56 +0400 Subject: [PATCH 276/279] QUIC: using QUIC API introduced in OpenSSL 3.5. Similarly to the QUIC API originated in BoringSSL, this API allows to register custom TLS callbacks for an external QUIC implementation. See the SSL_set_quic_tls_cbs manual page for details. Due to a different approach used in OpenSSL 3.5, handling of CRYPTO frames was streamlined to always write an incoming CRYPTO buffer to the crypto context. Using SSL_provide_quic_data(), this results in transient allocation of chain links and buffers for CRYPTO frames received in order. Testing didn't reveal performance degradation of QUIC handshakes, https://github.com/nginx/nginx/pull/646 provides specific results. --- auto/lib/openssl/conf | 10 +- src/event/quic/ngx_event_quic.h | 5 +- src/event/quic/ngx_event_quic_connection.h | 5 + src/event/quic/ngx_event_quic_ssl.c | 411 ++++++++++++++++++--- 4 files changed, 383 insertions(+), 48 deletions(-) diff --git a/auto/lib/openssl/conf b/auto/lib/openssl/conf index f4b00ebd672..3068cae36b1 100644 --- a/auto/lib/openssl/conf +++ b/auto/lib/openssl/conf @@ -147,11 +147,17 @@ else if [ $USE_OPENSSL_QUIC = YES ]; then - ngx_feature="OpenSSL QUIC support" + ngx_feature="OpenSSL QUIC API" ngx_feature_name="NGX_QUIC" - ngx_feature_test="SSL_set_quic_method(NULL, NULL)" + ngx_feature_test="SSL_set_quic_tls_cbs(NULL, NULL, NULL)" . auto/feature + if [ $ngx_found = no ]; then + ngx_feature="BoringSSL-like QUIC API" + ngx_feature_test="SSL_set_quic_method(NULL, NULL)" + . auto/feature + fi + if [ $ngx_found = no ]; then ngx_feature="OpenSSL QUIC compatibility" ngx_feature_test="SSL_CTX_add_custom_ext(NULL, 0, 0, diff --git a/src/event/quic/ngx_event_quic.h b/src/event/quic/ngx_event_quic.h index 50a5c214e15..d95d3d85b19 100644 --- a/src/event/quic/ngx_event_quic.h +++ b/src/event/quic/ngx_event_quic.h @@ -12,7 +12,10 @@ #include -#ifdef SSL_R_MISSING_QUIC_TRANSPORT_PARAMETERS_EXTENSION +#ifdef OSSL_RECORD_PROTECTION_LEVEL_NONE +#define NGX_QUIC_OPENSSL_API 1 + +#elif (defined SSL_R_MISSING_QUIC_TRANSPORT_PARAMETERS_EXTENSION) #define NGX_QUIC_QUICTLS_API 1 #elif (defined OPENSSL_IS_BORINGSSL || defined LIBRESSL_VERSION_NUMBER) diff --git a/src/event/quic/ngx_event_quic_connection.h b/src/event/quic/ngx_event_quic_connection.h index 856512118f9..33922cf801e 100644 --- a/src/event/quic/ngx_event_quic_connection.h +++ b/src/event/quic/ngx_event_quic_connection.h @@ -301,6 +301,11 @@ struct ngx_quic_connection_s { unsigned key_phase:1; unsigned validated:1; unsigned client_tp_done:1; + +#if (NGX_QUIC_OPENSSL_API) + unsigned read_level:2; + unsigned write_level:2; +#endif }; diff --git a/src/event/quic/ngx_event_quic_ssl.c b/src/event/quic/ngx_event_quic_ssl.c index 6ce926c8187..e961c80cd60 100644 --- a/src/event/quic/ngx_event_quic_ssl.c +++ b/src/event/quic/ngx_event_quic_ssl.c @@ -18,6 +18,23 @@ #define NGX_QUIC_MAX_BUFFERED 65535 +#if (NGX_QUIC_OPENSSL_API) + +static int ngx_quic_cbs_send(ngx_ssl_conn_t *ssl_conn, + const unsigned char *data, size_t len, size_t *consumed, void *arg); +static int ngx_quic_cbs_recv_rcd(ngx_ssl_conn_t *ssl_conn, + const unsigned char **data, size_t *bytes_read, void *arg); +static int ngx_quic_cbs_release_rcd(ngx_ssl_conn_t *ssl_conn, + size_t bytes_read, void *arg); +static int ngx_quic_cbs_yield_secret(ngx_ssl_conn_t *ssl_conn, uint32_t level, + int direction, const unsigned char *secret, size_t secret_len, void *arg); +static int ngx_quic_cbs_got_transport_params(ngx_ssl_conn_t *ssl_conn, + const unsigned char *params, size_t params_len, void *arg); +static int ngx_quic_cbs_alert(ngx_ssl_conn_t *ssl_conn, unsigned char alert, + void *arg); + +#else /* NGX_QUIC_BORINGSSL_API || NGX_QUIC_QUICTLS_API */ + static ngx_inline ngx_uint_t ngx_quic_map_encryption_level( enum ssl_encryption_level_t ssl_level); @@ -39,9 +56,270 @@ static int ngx_quic_add_handshake_data(ngx_ssl_conn_t *ssl_conn, static int ngx_quic_flush_flight(ngx_ssl_conn_t *ssl_conn); static int ngx_quic_send_alert(ngx_ssl_conn_t *ssl_conn, enum ssl_encryption_level_t ssl_level, uint8_t alert); + +#endif + static ngx_int_t ngx_quic_handshake(ngx_connection_t *c); -static ngx_int_t ngx_quic_crypto_provide(ngx_connection_t *c, ngx_chain_t *out, - ngx_uint_t level); +static ngx_int_t ngx_quic_crypto_provide(ngx_connection_t *c, ngx_uint_t level); + + +#if (NGX_QUIC_OPENSSL_API) + +static int +ngx_quic_cbs_send(ngx_ssl_conn_t *ssl_conn, + const unsigned char *data, size_t len, size_t *consumed, void *arg) +{ + ngx_connection_t *c = arg; + + ngx_chain_t *out; + unsigned int alpn_len; + ngx_quic_frame_t *frame; + const unsigned char *alpn_data; + ngx_quic_send_ctx_t *ctx; + ngx_quic_connection_t *qc; + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic ngx_quic_cbs_send len:%uz", len); + + qc = ngx_quic_get_connection(c); + + *consumed = 0; + + SSL_get0_alpn_selected(ssl_conn, &alpn_data, &alpn_len); + + if (alpn_len == 0) { + qc->error = NGX_QUIC_ERR_CRYPTO(SSL_AD_NO_APPLICATION_PROTOCOL); + qc->error_reason = "missing ALPN extension"; + + ngx_log_error(NGX_LOG_INFO, c->log, 0, + "quic missing ALPN extension"); + return 1; + } + + if (!qc->client_tp_done) { + /* RFC 9001, 8.2. QUIC Transport Parameters Extension */ + qc->error = NGX_QUIC_ERR_CRYPTO(SSL_AD_MISSING_EXTENSION); + qc->error_reason = "missing transport parameters"; + + ngx_log_error(NGX_LOG_INFO, c->log, 0, + "missing transport parameters"); + return 1; + } + + ctx = ngx_quic_get_send_ctx(qc, qc->write_level); + + out = ngx_quic_copy_buffer(c, (u_char *) data, len); + if (out == NGX_CHAIN_ERROR) { + qc->error = NGX_QUIC_ERR_INTERNAL_ERROR; + return 1; + } + + frame = ngx_quic_alloc_frame(c); + if (frame == NULL) { + qc->error = NGX_QUIC_ERR_INTERNAL_ERROR; + return 1; + } + + frame->data = out; + frame->level = qc->write_level; + frame->type = NGX_QUIC_FT_CRYPTO; + frame->u.crypto.offset = ctx->crypto_sent; + frame->u.crypto.length = len; + + ctx->crypto_sent += len; + *consumed = len; + + ngx_quic_queue_frame(qc, frame); + + return 1; +} + + +static int +ngx_quic_cbs_recv_rcd(ngx_ssl_conn_t *ssl_conn, + const unsigned char **data, size_t *bytes_read, void *arg) +{ + ngx_connection_t *c = arg; + + ngx_buf_t *b; + ngx_chain_t *cl; + ngx_quic_send_ctx_t *ctx; + ngx_quic_connection_t *qc; + + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic ngx_quic_cbs_recv_rcd"); + + qc = ngx_quic_get_connection(c); + ctx = ngx_quic_get_send_ctx(qc, qc->read_level); + + for (cl = ctx->crypto.chain; cl; cl = cl->next) { + b = cl->buf; + + if (b->sync) { + /* hole */ + + *bytes_read = 0; + + break; + } + + *data = b->pos; + *bytes_read = b->last - b->pos; + + break; + } + + return 1; +} + + +static int +ngx_quic_cbs_release_rcd(ngx_ssl_conn_t *ssl_conn, size_t bytes_read, void *arg) +{ + ngx_connection_t *c = arg; + + ngx_chain_t *cl; + ngx_quic_send_ctx_t *ctx; + ngx_quic_connection_t *qc; + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic ngx_quic_cbs_release_rcd len:%uz", bytes_read); + + qc = ngx_quic_get_connection(c); + ctx = ngx_quic_get_send_ctx(qc, qc->read_level); + + cl = ngx_quic_read_buffer(c, &ctx->crypto, bytes_read); + if (cl == NGX_CHAIN_ERROR) { + qc->error = NGX_QUIC_ERR_INTERNAL_ERROR; + return 1; + } + + ngx_quic_free_chain(c, cl); + + return 1; +} + + +static int +ngx_quic_cbs_yield_secret(ngx_ssl_conn_t *ssl_conn, uint32_t ssl_level, + int direction, const unsigned char *secret, size_t secret_len, void *arg) +{ + ngx_connection_t *c = arg; + + ngx_uint_t level; + const SSL_CIPHER *cipher; + ngx_quic_connection_t *qc; + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic ngx_quic_cbs_yield_secret() level:%uD", ssl_level); +#ifdef NGX_QUIC_DEBUG_CRYPTO + ngx_log_debug4(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic %s secret len:%uz %*xs", + direction ? "write" : "read", secret_len, + secret_len, secret); +#endif + + qc = ngx_quic_get_connection(c); + cipher = SSL_get_current_cipher(ssl_conn); + + switch (ssl_level) { + case OSSL_RECORD_PROTECTION_LEVEL_NONE: + level = NGX_QUIC_ENCRYPTION_INITIAL; + break; + case OSSL_RECORD_PROTECTION_LEVEL_EARLY: + level = NGX_QUIC_ENCRYPTION_EARLY_DATA; + break; + case OSSL_RECORD_PROTECTION_LEVEL_HANDSHAKE: + level = NGX_QUIC_ENCRYPTION_HANDSHAKE; + break; + default: /* OSSL_RECORD_PROTECTION_LEVEL_APPLICATION */ + level = NGX_QUIC_ENCRYPTION_APPLICATION; + break; + } + + if (ngx_quic_keys_set_encryption_secret(c->log, direction, qc->keys, level, + cipher, secret, secret_len) + != NGX_OK) + { + qc->error = NGX_QUIC_ERR_INTERNAL_ERROR; + return 1; + } + + if (direction) { + qc->write_level = level; + + } else { + qc->read_level = level; + } + + return 1; +} + + +static int +ngx_quic_cbs_got_transport_params(ngx_ssl_conn_t *ssl_conn, + const unsigned char *params, size_t params_len, void *arg) +{ + ngx_connection_t *c = arg; + + u_char *p, *end; + ngx_quic_tp_t ctp; + ngx_quic_connection_t *qc; + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic ngx_quic_cbs_got_transport_params() len:%uz", + params_len); + + qc = ngx_quic_get_connection(c); + + /* defaults for parameters not sent by client */ + ngx_memcpy(&ctp, &qc->ctp, sizeof(ngx_quic_tp_t)); + + p = (u_char *) params; + end = p + params_len; + + if (ngx_quic_parse_transport_params(p, end, &ctp, c->log) != NGX_OK) { + qc->error = NGX_QUIC_ERR_TRANSPORT_PARAMETER_ERROR; + qc->error_reason = "failed to process transport parameters"; + + return 1; + } + + if (ngx_quic_apply_transport_params(c, &ctp) != NGX_OK) { + return 1; + } + + qc->client_tp_done = 1; + + return 1; +} + + +static int +ngx_quic_cbs_alert(ngx_ssl_conn_t *ssl_conn, unsigned char alert, void *arg) +{ + ngx_connection_t *c = arg; + + ngx_quic_connection_t *qc; + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic ngx_quic_cbs_alert() alert:%d", (int) alert); + + /* already closed on regular shutdown */ + + qc = ngx_quic_get_connection(c); + if (qc == NULL) { + return 1; + } + + qc->error = NGX_QUIC_ERR_CRYPTO(alert); + qc->error_reason = "handshake failed"; + + return 1; +} + + +#else /* NGX_QUIC_BORINGSSL_API || NGX_QUIC_QUICTLS_API */ static ngx_inline ngx_uint_t @@ -340,13 +618,14 @@ ngx_quic_send_alert(ngx_ssl_conn_t *ssl_conn, return 1; } +#endif + ngx_int_t ngx_quic_handle_crypto_frame(ngx_connection_t *c, ngx_quic_header_t *pkt, ngx_quic_frame_t *frame) { uint64_t last; - ngx_chain_t *cl; ngx_quic_send_ctx_t *ctx; ngx_quic_connection_t *qc; ngx_quic_crypto_frame_t *f; @@ -385,41 +664,18 @@ ngx_quic_handle_crypto_frame(ngx_connection_t *c, ngx_quic_header_t *pkt, return NGX_OK; } - if (f->offset == ctx->crypto.offset) { - if (ngx_quic_crypto_provide(c, frame->data, pkt->level) != NGX_OK) { - return NGX_ERROR; - } - - if (ngx_quic_handshake(c) != NGX_OK) { - return NGX_ERROR; - } - - ngx_quic_skip_buffer(c, &ctx->crypto, last); - - } else { - if (ngx_quic_write_buffer(c, &ctx->crypto, frame->data, f->length, - f->offset) - == NGX_CHAIN_ERROR) - { - return NGX_ERROR; - } + if (ngx_quic_write_buffer(c, &ctx->crypto, frame->data, f->length, + f->offset) + == NGX_CHAIN_ERROR) + { + return NGX_ERROR; } - cl = ngx_quic_read_buffer(c, &ctx->crypto, (uint64_t) -1); - - if (cl) { - if (ngx_quic_crypto_provide(c, cl, pkt->level) != NGX_OK) { - return NGX_ERROR; - } - - if (ngx_quic_handshake(c) != NGX_OK) { - return NGX_ERROR; - } - - ngx_quic_free_chain(c, cl); + if (ngx_quic_crypto_provide(c, pkt->level) != NGX_OK) { + return NGX_ERROR; } - return NGX_OK; + return ngx_quic_handshake(c); } @@ -528,13 +784,24 @@ ngx_quic_handshake(ngx_connection_t *c) static ngx_int_t -ngx_quic_crypto_provide(ngx_connection_t *c, ngx_chain_t *out, - ngx_uint_t level) +ngx_quic_crypto_provide(ngx_connection_t *c, ngx_uint_t level) { +#if (NGX_QUIC_BORINGSSL_API || NGX_QUIC_QUICTLS_API) + ngx_buf_t *b; - ngx_chain_t *cl; + ngx_chain_t *out, *cl; + ngx_quic_send_ctx_t *ctx; + ngx_quic_connection_t *qc; enum ssl_encryption_level_t ssl_level; + qc = ngx_quic_get_connection(c); + ctx = ngx_quic_get_send_ctx(qc, level); + + out = ngx_quic_read_buffer(c, &ctx->crypto, (uint64_t) -1); + if (out == NGX_CHAIN_ERROR) { + return NGX_ERROR; + } + switch (level) { case NGX_QUIC_ENCRYPTION_INITIAL: ssl_level = ssl_encryption_initial; @@ -562,6 +829,10 @@ ngx_quic_crypto_provide(ngx_connection_t *c, ngx_chain_t *out, } } + ngx_quic_free_chain(c, out); + +#endif + return NGX_OK; } @@ -569,14 +840,40 @@ ngx_quic_crypto_provide(ngx_connection_t *c, ngx_chain_t *out, ngx_int_t ngx_quic_init_connection(ngx_connection_t *c) { - u_char *p; - size_t clen; - ssize_t len; - ngx_str_t dcid; - ngx_ssl_conn_t *ssl_conn; - ngx_quic_socket_t *qsock; - ngx_quic_connection_t *qc; - static SSL_QUIC_METHOD quic_method; + u_char *p; + size_t clen; + ssize_t len; + ngx_str_t dcid; + ngx_ssl_conn_t *ssl_conn; + ngx_quic_socket_t *qsock; + ngx_quic_connection_t *qc; + +#if (NGX_QUIC_OPENSSL_API) + static const OSSL_DISPATCH qtdis[] = { + + { OSSL_FUNC_SSL_QUIC_TLS_CRYPTO_SEND, + (void (*)(void)) ngx_quic_cbs_send }, + + { OSSL_FUNC_SSL_QUIC_TLS_CRYPTO_RECV_RCD, + (void (*)(void)) ngx_quic_cbs_recv_rcd }, + + { OSSL_FUNC_SSL_QUIC_TLS_CRYPTO_RELEASE_RCD, + (void (*)(void)) ngx_quic_cbs_release_rcd }, + + { OSSL_FUNC_SSL_QUIC_TLS_YIELD_SECRET, + (void (*)(void)) ngx_quic_cbs_yield_secret }, + + { OSSL_FUNC_SSL_QUIC_TLS_GOT_TRANSPORT_PARAMS, + (void (*)(void)) ngx_quic_cbs_got_transport_params }, + + { OSSL_FUNC_SSL_QUIC_TLS_ALERT, + (void (*)(void)) ngx_quic_cbs_alert }, + + { 0, NULL } + }; +#else /* NGX_QUIC_BORINGSSL_API || NGX_QUIC_QUICTLS_API */ + static SSL_QUIC_METHOD quic_method; +#endif qc = ngx_quic_get_connection(c); @@ -588,6 +885,20 @@ ngx_quic_init_connection(ngx_connection_t *c) ssl_conn = c->ssl->connection; +#if (NGX_QUIC_OPENSSL_API) + + if (SSL_set_quic_tls_cbs(ssl_conn, qtdis, c) == 0) { + ngx_ssl_error(NGX_LOG_ALERT, c->log, 0, + "quic SSL_set_quic_tls_cbs() failed"); + return NGX_ERROR; + } + + if (SSL_CTX_get_max_early_data(qc->conf->ssl->ctx)) { + SSL_set_quic_tls_early_data_enabled(ssl_conn, 1); + } + +#else /* NGX_QUIC_BORINGSSL_API || NGX_QUIC_QUICTLS_API */ + if (!quic_method.send_alert) { #if (NGX_QUIC_BORINGSSL_API) quic_method.set_read_secret = ngx_quic_set_read_secret; @@ -610,6 +921,8 @@ ngx_quic_init_connection(ngx_connection_t *c) if (SSL_CTX_get_max_early_data(qc->conf->ssl->ctx)) { SSL_set_quic_early_data_enabled(ssl_conn, 1); } +#endif + #endif qsock = ngx_quic_get_socket(c); @@ -641,11 +954,19 @@ ngx_quic_init_connection(ngx_connection_t *c) "quic transport parameters len:%uz %*xs", len, len, p); #endif +#if (NGX_QUIC_OPENSSL_API) + if (SSL_set_quic_tls_transport_params(ssl_conn, p, len) == 0) { + ngx_ssl_error(NGX_LOG_ALERT, c->log, 0, + "quic SSL_set_quic_tls_transport_params() failed"); + return NGX_ERROR; + } +#else if (SSL_set_quic_transport_params(ssl_conn, p, len) == 0) { ngx_ssl_error(NGX_LOG_ALERT, c->log, 0, "quic SSL_set_quic_transport_params() failed"); return NGX_ERROR; } +#endif #ifdef OPENSSL_IS_BORINGSSL if (SSL_set_quic_early_data_context(ssl_conn, p, clen) == 0) { From 0fdbfc1ff45adb8e98e71004e5d147987e7d8974 Mon Sep 17 00:00:00 2001 From: Aleksei Bavshin Date: Mon, 16 Dec 2024 17:56:45 -0800 Subject: [PATCH 277/279] SSL: support loading keys via OSSL_STORE. A new "store:..." prefix for the "ssl_certificate_key" directive allows loading keys via the OSSL_STORE API. The change is required to support hardware backed keys in OpenSSL 3.x using the new "provider(7ossl)" modules, such as "pkcs11-provider". While the engine API is present in 3.x, some operating systems (notably, RHEL10) have already disabled it in their builds of OpenSSL. Related: https://trac.nginx.org/nginx/ticket/2449 --- src/event/ngx_event_openssl_cache.c | 86 +++++++++++++++++++++++++++-- 1 file changed, 81 insertions(+), 5 deletions(-) diff --git a/src/event/ngx_event_openssl_cache.c b/src/event/ngx_event_openssl_cache.c index d62b4c4309c..cbb05892f0b 100644 --- a/src/event/ngx_event_openssl_cache.c +++ b/src/event/ngx_event_openssl_cache.c @@ -8,10 +8,16 @@ #include #include +#ifdef ERR_R_OSSL_STORE_LIB +#include +#include +#endif + #define NGX_SSL_CACHE_PATH 0 #define NGX_SSL_CACHE_DATA 1 #define NGX_SSL_CACHE_ENGINE 2 +#define NGX_SSL_CACHE_STORE 3 #define NGX_SSL_CACHE_DISABLED (ngx_array_t *) (uintptr_t) -1 @@ -444,6 +450,11 @@ ngx_ssl_cache_init_key(ngx_pool_t *pool, ngx_uint_t index, ngx_str_t *path, { id->type = NGX_SSL_CACHE_ENGINE; + } else if (index == NGX_SSL_CACHE_PKEY + && ngx_strncmp(path->data, "store:", sizeof("store:") - 1) == 0) + { + id->type = NGX_SSL_CACHE_STORE; + } else { if (ngx_get_full_name(pool, (ngx_str_t *) &ngx_cycle->conf_prefix, path) != NGX_OK) @@ -714,11 +725,6 @@ ngx_ssl_cache_pkey_create(ngx_ssl_cache_key_t *id, char **err, void *data) #endif } - bio = ngx_ssl_cache_create_bio(id, err); - if (bio == NULL) { - return NULL; - } - cb_data.encrypted = 0; if (*passwords) { @@ -734,6 +740,76 @@ ngx_ssl_cache_pkey_create(ngx_ssl_cache_key_t *id, char **err, void *data) cb = NULL; } + if (id->type == NGX_SSL_CACHE_STORE) { + +#ifdef ERR_R_OSSL_STORE_LIB + + u_char *uri; + UI_METHOD *method; + OSSL_STORE_CTX *store; + OSSL_STORE_INFO *info; + + method = (cb != NULL) ? UI_UTIL_wrap_read_pem_callback(cb, 0) : NULL; + uri = id->data + sizeof("store:") - 1; + + store = OSSL_STORE_open((char *) uri, method, pwd, NULL, NULL); + + if (store == NULL) { + *err = "OSSL_STORE_open() failed"; + + if (method != NULL) { + UI_destroy_method(method); + } + + return NULL; + } + + pkey = NULL; + + while (pkey == NULL && !OSSL_STORE_eof(store)) { + info = OSSL_STORE_load(store); + + if (info == NULL) { + continue; + } + + if (OSSL_STORE_INFO_get_type(info) == OSSL_STORE_INFO_PKEY) { + pkey = OSSL_STORE_INFO_get1_PKEY(info); + } + + OSSL_STORE_INFO_free(info); + } + + OSSL_STORE_close(store); + + if (method != NULL) { + UI_destroy_method(method); + } + + if (pkey == NULL) { + *err = "OSSL_STORE_load() failed"; + return NULL; + } + + if (cb_data.encrypted) { + *passwords = NGX_SSL_CACHE_DISABLED; + } + + return pkey; + +#else + + *err = "loading \"store:...\" certificate keys is not supported"; + return NULL; + +#endif + } + + bio = ngx_ssl_cache_create_bio(id, err); + if (bio == NULL) { + return NULL; + } + for ( ;; ) { pkey = PEM_read_bio_PrivateKey(bio, NULL, cb, pwd); From 3d5889a3ee41a282bad54d9c0d3662dba9f52c1b Mon Sep 17 00:00:00 2001 From: Aleksei Bavshin Date: Fri, 17 Jan 2025 12:24:08 -0800 Subject: [PATCH 278/279] SSL: disabled UI console prompts from worker processes. Certain providers may attempt to reload the key on the first use after a fork. Such attempt would require re-prompting the pin, and this time we are not able to pass the password callback. While it is addressable with configuration for a specific provider, it would be prudent to ensure that no such prompts could block worker processes by setting the default UI method. UI_null() first appeared in 1.1.1 along with the OSSL_STORE, so it is safe to assume the same set of guards. --- src/event/ngx_event_openssl_cache.c | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/event/ngx_event_openssl_cache.c b/src/event/ngx_event_openssl_cache.c index cbb05892f0b..18efc73d06b 100644 --- a/src/event/ngx_event_openssl_cache.c +++ b/src/event/ngx_event_openssl_cache.c @@ -122,6 +122,8 @@ static void ngx_ssl_cache_node_insert(ngx_rbtree_node_t *temp, static void ngx_ssl_cache_node_free(ngx_rbtree_t *rbtree, ngx_ssl_cache_node_t *cn); +static ngx_int_t ngx_openssl_cache_init_worker(ngx_cycle_t *cycle); + static ngx_command_t ngx_openssl_cache_commands[] = { @@ -150,7 +152,7 @@ ngx_module_t ngx_openssl_cache_module = { NGX_CORE_MODULE, /* module type */ NULL, /* init master */ NULL, /* init module */ - NULL, /* init process */ + ngx_openssl_cache_init_worker, /* init process */ NULL, /* init thread */ NULL, /* exit thread */ NULL, /* exit process */ @@ -1233,3 +1235,20 @@ ngx_ssl_cache_node_insert(ngx_rbtree_node_t *temp, node->right = sentinel; ngx_rbt_red(node); } + + +static ngx_int_t +ngx_openssl_cache_init_worker(ngx_cycle_t *cycle) +{ +#ifdef ERR_R_OSSL_STORE_LIB + + if (ngx_process != NGX_PROCESS_WORKER) { + return NGX_OK; + } + + UI_set_default_method(UI_null()); + +#endif + + return NGX_OK; +} From 5b8a5c08ce28639e788734b2528faad70baa113c Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Mon, 26 May 2025 16:11:36 +0400 Subject: [PATCH 279/279] Core: added support for TCP keepalive parameters on macOS. The support first appeared in OS X Mavericks 10.9 and documented since OS X Yosemite 10.10. It has a subtle implementation difference from other operating systems in that the TCP_KEEPALIVE socket option (used in place of TCP_KEEPIDLE) isn't inherited from a listening socket to an accepted socket. An apparent reason for this behaviour is that it might be preserved for the sake of backward compatibility. The TCP_KEEPALIVE socket option is not inherited since appearance in OS X Panther 10.3, which long predates two other TCP_KEEPINTVL and TCP_KEEPCNT socket options. Thanks to Andy Pan for initial work. --- auto/os/darwin | 16 ++++++++++++++++ auto/unix | 26 ++++++++++++++------------ src/core/ngx_connection.c | 4 ++++ src/event/ngx_event_accept.c | 17 +++++++++++++++++ 4 files changed, 51 insertions(+), 12 deletions(-) diff --git a/auto/os/darwin b/auto/os/darwin index 429468f7fd4..0ede28d0ac7 100644 --- a/auto/os/darwin +++ b/auto/os/darwin @@ -118,3 +118,19 @@ ngx_feature_libs= ngx_feature_test="int32_t lock = 0; if (!OSAtomicCompareAndSwap32Barrier(0, 1, &lock)) return 1" . auto/feature + + +ngx_feature="TCP_KEEPALIVE" +ngx_feature_name="NGX_HAVE_KEEPALIVE_TUNABLE" +ngx_feature_run=no +ngx_feature_incs="#include + #include + #include " +ngx_feature_path= +ngx_feature_libs= +ngx_feature_test="setsockopt(0, IPPROTO_TCP, TCP_KEEPALIVE, NULL, 0); + setsockopt(0, IPPROTO_TCP, TCP_KEEPINTVL, NULL, 0); + setsockopt(0, IPPROTO_TCP, TCP_KEEPCNT, NULL, 0)" +. auto/feature + +NGX_KEEPALIVE_CHECKED=YES diff --git a/auto/unix b/auto/unix index 8bd1b1370dd..0dd66cfcd6f 100644 --- a/auto/unix +++ b/auto/unix @@ -508,18 +508,20 @@ ngx_feature_test="setsockopt(0, IPPROTO_TCP, TCP_DEFER_ACCEPT, NULL, 0)" . auto/feature -ngx_feature="TCP_KEEPIDLE" -ngx_feature_name="NGX_HAVE_KEEPALIVE_TUNABLE" -ngx_feature_run=no -ngx_feature_incs="#include - #include - #include " -ngx_feature_path= -ngx_feature_libs= -ngx_feature_test="setsockopt(0, IPPROTO_TCP, TCP_KEEPIDLE, NULL, 0); - setsockopt(0, IPPROTO_TCP, TCP_KEEPINTVL, NULL, 0); - setsockopt(0, IPPROTO_TCP, TCP_KEEPCNT, NULL, 0)" -. auto/feature +if test -z "$NGX_KEEPALIVE_CHECKED"; then + ngx_feature="TCP_KEEPIDLE" + ngx_feature_name="NGX_HAVE_KEEPALIVE_TUNABLE" + ngx_feature_run=no + ngx_feature_incs="#include + #include + #include " + ngx_feature_path= + ngx_feature_libs= + ngx_feature_test="setsockopt(0, IPPROTO_TCP, TCP_KEEPIDLE, NULL, 0); + setsockopt(0, IPPROTO_TCP, TCP_KEEPINTVL, NULL, 0); + setsockopt(0, IPPROTO_TCP, TCP_KEEPCNT, NULL, 0)" + . auto/feature +fi ngx_feature="TCP_FASTOPEN" diff --git a/src/core/ngx_connection.c b/src/core/ngx_connection.c index 75809d9ad96..7cae295eb48 100644 --- a/src/core/ngx_connection.c +++ b/src/core/ngx_connection.c @@ -765,6 +765,8 @@ ngx_configure_listening_sockets(ngx_cycle_t *cycle) #if (NGX_HAVE_KEEPALIVE_TUNABLE) +#if !(NGX_DARWIN) + if (ls[i].keepidle) { value = ls[i].keepidle; @@ -782,6 +784,8 @@ ngx_configure_listening_sockets(ngx_cycle_t *cycle) } } +#endif + if (ls[i].keepintvl) { value = ls[i].keepintvl; diff --git a/src/event/ngx_event_accept.c b/src/event/ngx_event_accept.c index 27038799de5..033d7e021ec 100644 --- a/src/event/ngx_event_accept.c +++ b/src/event/ngx_event_accept.c @@ -203,6 +203,23 @@ ngx_event_accept(ngx_event_t *ev) } } +#if (NGX_HAVE_KEEPALIVE_TUNABLE && NGX_DARWIN) + + /* Darwin doesn't inherit TCP_KEEPALIVE from a listening socket */ + + if (ls->keepidle) { + if (setsockopt(s, IPPROTO_TCP, TCP_KEEPALIVE, + (const void *) &ls->keepidle, sizeof(int)) + == -1) + { + ngx_log_error(NGX_LOG_ALERT, ev->log, ngx_socket_errno, + "setsockopt(TCP_KEEPALIVE, %d) failed, ignored", + ls->keepidle); + } + } + +#endif + *log = ls->log; c->recv = ngx_recv;