diff --git a/kmod/toa/example_nat64/nginx/README.md b/kmod/toa/example_nat64/nginx/README.md deleted file mode 100644 index 3e3190ee9..000000000 --- a/kmod/toa/example_nat64/nginx/README.md +++ /dev/null @@ -1,32 +0,0 @@ -This patch is for Nginx to get real client ip by 'toa_remote_addr' -when you are using NAT64 mode(VIP is IPv6 while RS is IPv4). -You can use this patch only when toa module is installed. - -Here is an exampe to configure http block in nginx.conf: - -``` -http { - include mime.types; - default_type application/octet-stream; - - log_format main '$toa_remote_addr $toa_remote_port $remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; - - access_log /data/nginx/logs/access.log main; - - keepalive_timeout 65; - - server { - listen 80; - server_name localhost; - - access_log /data/nginx/logs/access.log main; - - location / { - proxy_set_header X-Forwarded-For $toa_remote_addr; - proxy_pass http://192.168.1.1; - } - } -} -``` diff --git a/patch/nginx/README.md b/patch/nginx/README.md new file mode 100644 index 000000000..2614a3f5a --- /dev/null +++ b/patch/nginx/README.md @@ -0,0 +1,80 @@ +Nginx Patches for DPVS +----- + +The directory is arranged to place nginx patch files for DPVS. More specifically, it contains the following patches. + +* TOA patch for originating client IP/port derived from DPVS NAT64 translation +* UOA patch for originating client IP/port derived from DPVS UDP FNAT/NAT64 translation in QUIC/HTTP3 +* QUIC Server Connection ID patch for connection migration + +## TOA NAT64 + +Nginx can get the originating client IP address and Port NAT64'ed by DPVS by utilizing nginx variables 'toa_remote_addr' and 'toa_remote_port' respectively. It works when and only when the TOA kernel module has already installed successfully on the nginx server. + +This is an exampe configuration of nginx with TOA patch for NAT64. + +``` +http { + include mime.types; + default_type application/octet-stream; + + log_format nat64 '$remote_addr $toa_remote_addr :$toa_remote_port - $remote_user [$time_local] ' + '"$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" "$http_x_forwarded_for" ' + '$request_length $upstream_response_time $upstream_addr'; + + access_log logs/access.log nat64; + + # more other configs ...... + +} +``` + +## UOA QUIC/HTTP3 + +Nginx can get the originating client IP address and Port NAT'ed by DPVS by utilizing nginx variables 'uoa_remote_addr' and 'uoa_remote_port' respectively. Both IPv4-IPv4 and IPv6-IPv6 NAT and NAT64(IPv6-IPv4 NAT) as well are supported. It works when and only when the UOA kernel module has already installed sucessfully on the nginx server. + +This is an exampe configuration of nginx with UOA patch. + +``` +http { + include mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] ' + '"$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" "$http3" ' + '"$http_x_forwarded_for" $request_length $upstream_response_time $upstream_addr'; + + log_format quic '$remote_addr $uoa_remote_addr :$uoa_remote_port - $remote_user [$time_local] ' + '"$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" "$http3" ' + '"$http_x_forwarded_for" $request_length $upstream_response_time $upstream_addr'; + + access_log logs/access.log main; + + # more other configs ...... + + + server { + listen 443 quic reuseport; + listen 443 ssl; + + server_name qlb-test.qiyi.domain; + + access_log logs/quic.access.log quic; + + ssl_certificate certs/cert.pem; + ssl_certificate_key certs/key.pem; + + location / { + add_header Alt-Svc 'h3=":2443"; ma=86400'; + root html; + index index.html index.htm; + } + } +} +``` + +## Quic Server Connection ID + +It requires changes to Quic Server Connection ID(SCID) both in DPVS and Nginx to support the feature of QUIC connection migration. DPVS depends on Server IP/Port information encoded in SCID to schedule a migrating connection to the right nginx server where the previous connection resides, and Nginx relies on the socket cookie compiled in SCID to make a migrating connection be processed on the same listening socket as the previous one. Note that eBPF (bpf_sk_select_reuseport) is used in Nginx for QUIC connection migration, which requires Linux 5.7+. + +The patch adds Nginx server address information into SCID, and fixes its collision problem with Nginx's socket cookie. The server address contains 24 least significant bits(LSB) for IPv4, and 32 LSB for IPv6, and compliant with DPVS DCID format specification defined in [ipvs/quic.h](../../include/ipvs/quic.h). The server port is not included in SCID. diff --git a/kmod/toa/example_nat64/nginx/nginx-1.14.0-nat64-toa.patch b/patch/nginx/v1.14.0/nginx-1.14.0-nat64-toa.patch similarity index 100% rename from kmod/toa/example_nat64/nginx/nginx-1.14.0-nat64-toa.patch rename to patch/nginx/v1.14.0/nginx-1.14.0-nat64-toa.patch diff --git a/patch/nginx/v1.26.0/0001-patch-toa-for-nat64-and-uoa-for-quic.patch b/patch/nginx/v1.26.0/0001-patch-toa-for-nat64-and-uoa-for-quic.patch new file mode 100644 index 000000000..8f9838523 --- /dev/null +++ b/patch/nginx/v1.26.0/0001-patch-toa-for-nat64-and-uoa-for-quic.patch @@ -0,0 +1,398 @@ +From b5137f7826d2fa85352cbb1afe5f2317a4f3abd1 Mon Sep 17 00:00:00 2001 +From: wencyu +Date: Fri, 24 May 2024 16:37:06 +0800 +Subject: [PATCH 1/7] patch: toa for nat64 and uoa for quic + +Signed-off-by: wencyu +--- + src/core/ngx_connection.h | 5 ++ + src/core/ngx_inet.h | 44 +++++++++++ + src/event/ngx_event_accept.c | 23 +++++- + src/event/quic/ngx_event_quic_streams.c | 15 +++- + src/event/quic/ngx_event_quic_udp.c | 38 ++++++++- + src/http/ngx_http_variables.c | 133 ++++++++++++++++++++++++++++++++ + 6 files changed, 255 insertions(+), 3 deletions(-) + +diff --git a/src/core/ngx_connection.h b/src/core/ngx_connection.h +index 84dd804..0ce4c69 100644 +--- a/src/core/ngx_connection.h ++++ b/src/core/ngx_connection.h +@@ -147,6 +147,11 @@ struct ngx_connection_s { + socklen_t socklen; + ngx_str_t addr_text; + ++ union { ++ struct sockaddr *toa_addr; ++ struct sockaddr *uoa_addr; ++ }; ++ + ngx_proxy_protocol_t *proxy_protocol; + + #if (NGX_QUIC || NGX_COMPAT) +diff --git a/src/core/ngx_inet.h b/src/core/ngx_inet.h +index 19050fc..3365c8e 100644 +--- a/src/core/ngx_inet.h ++++ b/src/core/ngx_inet.h +@@ -106,6 +106,50 @@ typedef struct { + } ngx_url_t; + + ++/* toa socket options, now only for nat64 */ ++enum { ++ TOA_BASE_CTL = 4096, ++ /* set */ ++ TOA_SO_SET_MAX = TOA_BASE_CTL, ++ /* get */ ++ TOA_SO_GET_LOOKUP = TOA_BASE_CTL, ++ TOA_SO_GET_MAX = TOA_SO_GET_LOOKUP, ++}; ++ ++/* uoa socket options */ ++enum { ++ UOA_BASE_CTL = 2048, ++ /* set */ ++ UOA_SO_SET_MAX = UOA_BASE_CTL, ++ /* get */ ++ UOA_SO_GET_LOOKUP = UOA_BASE_CTL, ++ UOA_SO_GET_MAX = UOA_SO_GET_LOOKUP, ++}; ++ ++typedef struct { ++ struct in6_addr saddr; ++ uint16_t port; ++} toa_nat64_peer_t; ++ ++typedef union { ++ struct in_addr in; ++ struct in6_addr in6; ++} uoa_addr_t; ++ ++typedef struct { ++ /* input */ ++ uint16_t af; ++ uoa_addr_t saddr; ++ uoa_addr_t daddr; ++ uint16_t sport; ++ uint16_t dport; ++ /* output */ ++ uint16_t real_af; ++ uoa_addr_t real_saddr; ++ uint16_t real_sport; ++} __attribute__((__packed__)) uoa_param_map_t; ++ ++ + in_addr_t ngx_inet_addr(u_char *text, size_t len); + #if (NGX_HAVE_INET6) + ngx_int_t ngx_inet6_addr(u_char *p, size_t len, u_char *addr); +diff --git a/src/event/ngx_event_accept.c b/src/event/ngx_event_accept.c +index 2703879..66f28b2 100644 +--- a/src/event/ngx_event_accept.c ++++ b/src/event/ngx_event_accept.c +@@ -20,7 +20,7 @@ static void ngx_close_accepted_connection(ngx_connection_t *c); + void + ngx_event_accept(ngx_event_t *ev) + { +- socklen_t socklen; ++ socklen_t socklen, toa64_addrlen; + ngx_err_t err; + ngx_log_t *log; + ngx_uint_t level; +@@ -30,6 +30,7 @@ ngx_event_accept(ngx_event_t *ev) + ngx_listening_t *ls; + ngx_connection_t *c, *lc; + ngx_event_conf_t *ecf; ++ toa_nat64_peer_t toa64_addr; + #if (NGX_HAVE_ACCEPT4) + static ngx_uint_t use_accept4 = 1; + #endif +@@ -174,6 +175,26 @@ ngx_event_accept(ngx_event_t *ev) + + ngx_memcpy(c->sockaddr, &sa, socklen); + ++ ++ /* get nat64 toa remote addr & port */ ++ toa64_addrlen = sizeof(toa_nat64_peer_t); ++ if (getsockopt(s, IPPROTO_IP, TOA_SO_GET_LOOKUP, &toa64_addr, ++ &toa64_addrlen) == NGX_OK) { ++ struct sockaddr_in6 *toa64_sa; ++ c->toa_addr = ngx_palloc(c->pool, sizeof(struct sockaddr_in6)); ++ if (c->toa_addr == NULL) { ++ ngx_close_accepted_connection(c); ++ return; ++ } ++ ngx_memzero(c->toa_addr, sizeof(struct sockaddr_in6)); ++ toa64_sa = (struct sockaddr_in6 *)c->toa_addr; ++ toa64_sa->sin6_family = AF_INET6; ++ toa64_sa->sin6_addr = toa64_addr.saddr; ++ toa64_sa->sin6_port = toa64_addr.port; ++ } else { ++ c->toa_addr = NULL; ++ } ++ + log = ngx_palloc(c->pool, sizeof(ngx_log_t)); + if (log == NULL) { + ngx_close_accepted_connection(c); +diff --git a/src/event/quic/ngx_event_quic_streams.c b/src/event/quic/ngx_event_quic_streams.c +index 178b805..ce11cff 100644 +--- a/src/event/quic/ngx_event_quic_streams.c ++++ b/src/event/quic/ngx_event_quic_streams.c +@@ -646,7 +646,7 @@ ngx_quic_create_stream(ngx_connection_t *c, uint64_t id) + ngx_pool_t *pool; + ngx_uint_t reusable; + ngx_queue_t *q; +- struct sockaddr *sockaddr; ++ struct sockaddr *sockaddr, *uoa_addr; + ngx_connection_t *sc; + ngx_quic_stream_t *qs; + ngx_pool_cleanup_t *cln; +@@ -707,6 +707,18 @@ ngx_quic_create_stream(ngx_connection_t *c, uint64_t id) + + ngx_memcpy(sockaddr, c->sockaddr, c->socklen); + ++ if (c->uoa_addr) { ++ uoa_addr = ngx_palloc(pool, sizeof(struct sockaddr_storage)); ++ if (uoa_addr == NULL) { ++ ngx_destroy_pool(pool); ++ ngx_queue_insert_tail(&qc->streams.free, &qs->queue); ++ return NULL; ++ } ++ ngx_memcpy(uoa_addr, c->uoa_addr, sizeof(struct sockaddr_storage)); ++ } else { ++ uoa_addr = NULL; ++ } ++ + if (c->addr_text.data) { + addr_text.data = ngx_pnalloc(pool, c->addr_text.len); + if (addr_text.data == NULL) { +@@ -743,6 +755,7 @@ ngx_quic_create_stream(ngx_connection_t *c, uint64_t id) + sc->ssl = c->ssl; + sc->sockaddr = sockaddr; + sc->socklen = c->socklen; ++ sc->uoa_addr = uoa_addr; + sc->listening = c->listening; + sc->addr_text = addr_text; + sc->local_sockaddr = c->local_sockaddr; +diff --git a/src/event/quic/ngx_event_quic_udp.c b/src/event/quic/ngx_event_quic_udp.c +index 15b54bc..e94e0f5 100644 +--- a/src/event/quic/ngx_event_quic_udp.c ++++ b/src/event/quic/ngx_event_quic_udp.c +@@ -24,7 +24,7 @@ ngx_quic_recvmsg(ngx_event_t *ev) + ngx_buf_t buf; + ngx_log_t *log; + ngx_err_t err; +- socklen_t socklen, local_socklen; ++ socklen_t socklen, local_socklen, uoamap_len; + ngx_event_t *rev, *wev; + struct iovec iov[1]; + struct msghdr msg; +@@ -34,6 +34,7 @@ ngx_quic_recvmsg(ngx_event_t *ev) + ngx_event_conf_t *ecf; + ngx_connection_t *c, *lc; + ngx_quic_socket_t *qsock; ++ uoa_param_map_t uoamap; + static u_char buffer[NGX_QUIC_MAX_UDP_PAYLOAD_SIZE]; + + #if (NGX_HAVE_ADDRINFO_CMSG) +@@ -238,6 +239,41 @@ ngx_quic_recvmsg(ngx_event_t *ev) + + ngx_memcpy(c->sockaddr, sockaddr, socklen); + ++ /* parse uoa address */ ++ if (sockaddr->sa_family == AF_INET || sockaddr->sa_family == AF_INET6) { ++ uoamap_len = sizeof(uoamap); ++ ngx_memzero(&uoamap, uoamap_len); ++ uoamap.af = sockaddr->sa_family; ++ if (uoamap.af == AF_INET) { ++ uoamap.saddr.in = ((ngx_sockaddr_t *)sockaddr)->sockaddr_in.sin_addr; ++ uoamap.sport = ((ngx_sockaddr_t *)sockaddr)->sockaddr_in.sin_port; ++ uoamap.dport = ((ngx_sockaddr_t *)local_sockaddr)->sockaddr_in.sin_port; ++ } else { ++ uoamap.saddr.in6 = ((ngx_sockaddr_t *)sockaddr)->sockaddr_in6.sin6_addr; ++ uoamap.sport = ((ngx_sockaddr_t *)sockaddr)->sockaddr_in6.sin6_port; ++ uoamap.dport = ((ngx_sockaddr_t *)local_sockaddr)->sockaddr_in6.sin6_port; ++ } ++ if (getsockopt(lc->fd, IPPROTO_IP, UOA_SO_GET_LOOKUP, &uoamap, &uoamap_len) ++ == NGX_OK) { ++ c->uoa_addr = ngx_palloc(c->pool, sizeof(struct sockaddr_storage)); ++ if (c->uoa_addr == NULL) { ++ ngx_quic_close_accepted_connection(c); ++ return; ++ } ++ ngx_memzero(c->uoa_addr, sizeof(struct sockaddr_storage)); ++ c->uoa_addr->sa_family = uoamap.real_af; ++ if (uoamap.real_af == AF_INET) { ++ ((ngx_sockaddr_t *)c->uoa_addr)->sockaddr_in.sin_port = uoamap.real_sport; ++ ((ngx_sockaddr_t *)c->uoa_addr)->sockaddr_in.sin_addr = uoamap.real_saddr.in; ++ } else { ++ ((ngx_sockaddr_t *)c->uoa_addr)->sockaddr_in6.sin6_port = uoamap.real_sport; ++ ((ngx_sockaddr_t *)c->uoa_addr)->sockaddr_in6.sin6_addr = uoamap.real_saddr.in6; ++ } ++ } else { ++ c->uoa_addr = NULL; ++ } ++ } ++ + log = ngx_palloc(c->pool, sizeof(ngx_log_t)); + if (log == NULL) { + ngx_quic_close_accepted_connection(c); +diff --git a/src/http/ngx_http_variables.c b/src/http/ngx_http_variables.c +index 4f0bd0e..55c6bd7 100644 +--- a/src/http/ngx_http_variables.c ++++ b/src/http/ngx_http_variables.c +@@ -57,6 +57,14 @@ static ngx_int_t ngx_http_variable_remote_addr(ngx_http_request_t *r, + ngx_http_variable_value_t *v, uintptr_t data); + static ngx_int_t ngx_http_variable_remote_port(ngx_http_request_t *r, + ngx_http_variable_value_t *v, uintptr_t data); ++static ngx_int_t ngx_http_variable_toa_remote_addr(ngx_http_request_t *r, ++ ngx_http_variable_value_t *v, uintptr_t data); ++static ngx_int_t ngx_http_variable_toa_remote_port(ngx_http_request_t *r, ++ ngx_http_variable_value_t *v, uintptr_t data); ++static ngx_int_t ngx_http_variable_uoa_remote_addr(ngx_http_request_t *r, ++ ngx_http_variable_value_t *v, uintptr_t data); ++static ngx_int_t ngx_http_variable_uoa_remote_port(ngx_http_request_t *r, ++ ngx_http_variable_value_t *v, uintptr_t data); + static ngx_int_t ngx_http_variable_proxy_protocol_addr(ngx_http_request_t *r, + ngx_http_variable_value_t *v, uintptr_t data); + static ngx_int_t ngx_http_variable_proxy_protocol_port(ngx_http_request_t *r, +@@ -200,6 +208,14 @@ static ngx_http_variable_t ngx_http_core_variables[] = { + + { ngx_string("remote_port"), NULL, ngx_http_variable_remote_port, 0, 0, 0 }, + ++ { ngx_string("toa_remote_addr"), NULL, ngx_http_variable_toa_remote_addr, 0, 0, 0 }, ++ ++ { ngx_string("toa_remote_port"), NULL, ngx_http_variable_toa_remote_port, 0, 0, 0 }, ++ ++ { ngx_string("uoa_remote_addr"), NULL, ngx_http_variable_uoa_remote_addr, 0, 0, 0 }, ++ ++ { ngx_string("uoa_remote_port"), NULL, ngx_http_variable_uoa_remote_port, 0, 0, 0 }, ++ + { ngx_string("proxy_protocol_addr"), NULL, + ngx_http_variable_proxy_protocol_addr, + offsetof(ngx_proxy_protocol_t, src_addr), 0, 0 }, +@@ -1335,6 +1351,123 @@ ngx_http_variable_remote_port(ngx_http_request_t *r, + return NGX_OK; + } + ++static ngx_int_t ngx_http_variable_toa_remote_addr(ngx_http_request_t *r, ++ ngx_http_variable_value_t *v, uintptr_t data) { ++ struct sockaddr_in6 *sa6; ++ size_t len; ++ ++ len = r->connection->toa_addr ? NGX_INET6_ADDRSTRLEN : 1; ++ v->data = ngx_pnalloc(r->pool, len); ++ if (v->data == NULL) { ++ return NGX_ERROR; ++ } ++ ++ v->len = 0; ++ v->valid = 1; ++ v->no_cacheable = 0; ++ v->not_found = 0; ++ ++ if (r->connection->toa_addr) { ++ sa6 = (struct sockaddr_in6 *)r->connection->toa_addr; ++ v->len = ngx_inet_ntop(sa6->sin6_family, &sa6->sin6_addr, (u_char *)v->data, len); ++ } else { ++ v->data[0] = '-'; ++ v->len = 1; ++ } ++ ++ return NGX_OK; ++} ++ ++static ngx_int_t ngx_http_variable_toa_remote_port(ngx_http_request_t *r, ++ ngx_http_variable_value_t *v, uintptr_t data) { ++ ngx_uint_t port; ++ size_t len; ++ ++ len = r->connection->toa_addr ? sizeof("65535") - 1 : 1; ++ v->data = ngx_pnalloc(r->pool, len); ++ if (v->data == NULL) { ++ return NGX_ERROR; ++ } ++ ++ v->len = 0; ++ v->valid = 1; ++ v->no_cacheable = 0; ++ v->not_found = 0; ++ ++ if (r->connection->toa_addr) { ++ port = ngx_inet_get_port(r->connection->toa_addr); ++ if (port > 0 && port < 65536) { ++ v->len = ngx_sprintf(v->data, "%ui", port) - v->data; ++ } ++ } else { ++ v->data[0] = '-'; ++ v->len = 1; ++ } ++ ++ return NGX_OK; ++} ++ ++static ngx_int_t ngx_http_variable_uoa_remote_addr(ngx_http_request_t *r, ++ ngx_http_variable_value_t *v, uintptr_t data) { ++ struct sockaddr_in *sa; ++ struct sockaddr_in6 *sa6; ++ size_t len; ++ ++ len = r->connection->uoa_addr ? NGX_INET6_ADDRSTRLEN : 1; ++ v->data = ngx_pnalloc(r->pool, len); ++ if (v->data == NULL) { ++ return NGX_ERROR; ++ } ++ ++ v->len = 0; ++ v->valid = 1; ++ v->no_cacheable = 0; ++ v->not_found = 0; ++ ++ if (r->connection->uoa_addr) { ++ if (r->connection->uoa_addr->sa_family == AF_INET6) { ++ sa6 = (struct sockaddr_in6 *)r->connection->uoa_addr; ++ v->len = ngx_inet_ntop(sa6->sin6_family, &sa6->sin6_addr, (u_char *)v->data, len); ++ } else { ++ sa = (struct sockaddr_in *)r->connection->uoa_addr; ++ v->len = ngx_inet_ntop(sa->sin_family, &sa->sin_addr, (u_char *)v->data, len); ++ } ++ } else { ++ v->data[0] = '-'; ++ v->len = 1; ++ } ++ ++ return NGX_OK; ++} ++ ++static ngx_int_t ngx_http_variable_uoa_remote_port(ngx_http_request_t *r, ++ ngx_http_variable_value_t *v, uintptr_t data) { ++ ngx_uint_t port; ++ size_t len; ++ ++ len = r->connection->uoa_addr ? sizeof("65535") - 1 : 1; ++ v->data = ngx_pnalloc(r->pool, len); ++ if (v->data == NULL) { ++ return NGX_ERROR; ++ } ++ ++ v->len = 0; ++ v->valid = 1; ++ v->no_cacheable = 0; ++ v->not_found = 0; ++ ++ if (r->connection->uoa_addr) { ++ port = ngx_inet_get_port(r->connection->uoa_addr); ++ if (port > 0 && port < 65536) { ++ v->len = ngx_sprintf(v->data, "%ui", port) - v->data; ++ } ++ } else { ++ v->data[0] = '-'; ++ v->len = 1; ++ } ++ ++ return NGX_OK; ++} + + static ngx_int_t + ngx_http_variable_proxy_protocol_addr(ngx_http_request_t *r, +-- +1.8.3.1 + diff --git a/patch/nginx/v1.26.0/0002-patch-quic-connection-migration-for-l4lb-dpvs.patch b/patch/nginx/v1.26.0/0002-patch-quic-connection-migration-for-l4lb-dpvs.patch new file mode 100644 index 000000000..52962a860 --- /dev/null +++ b/patch/nginx/v1.26.0/0002-patch-quic-connection-migration-for-l4lb-dpvs.patch @@ -0,0 +1,248 @@ +From 832740515032f6169635f13eacc0fa50d5560d51 Mon Sep 17 00:00:00 2001 +From: wencyu +Date: Wed, 29 May 2024 10:20:09 +0800 +Subject: [PATCH 2/7] patch: quic connection migration for l4lb/dpvs + +Signed-off-by: wencyu +--- + src/event/quic/bpf/ngx_quic_reuseport_helper.c | 8 ++-- + src/event/quic/ngx_event_quic_bpf_code.c | 58 +++++++++++++----------- + src/event/quic/ngx_event_quic_connid.c | 62 ++++++++++++++++++++++++++ + src/event/quic/ngx_event_quic_transport.c | 1 + + 4 files changed, 101 insertions(+), 28 deletions(-) + +diff --git a/src/event/quic/bpf/ngx_quic_reuseport_helper.c b/src/event/quic/bpf/ngx_quic_reuseport_helper.c +index 999e760..bdca492 100644 +--- a/src/event/quic/bpf/ngx_quic_reuseport_helper.c ++++ b/src/event/quic/bpf/ngx_quic_reuseport_helper.c +@@ -76,7 +76,7 @@ int ngx_quic_select_socket_by_dcid(struct sk_reuseport_md *ctx) + int rc; + __u64 key; + size_t len, offset; +- unsigned char *start, *end, *data, *dcid; ++ unsigned char *start, *end, *data, *dcid, *cookie; + + start = ctx->data; + end = (unsigned char *) ctx->data_end; +@@ -104,12 +104,14 @@ int ngx_quic_select_socket_by_dcid(struct sk_reuseport_md *ctx) + dcid = &data[1]; + advance_data(len); /* we expect the packet to have full DCID */ + ++ cookie = dcid + (len - sizeof(__u64)); /* socket cookie is at the tail of DCID */ ++ + /* make verifier happy */ +- if (dcid + sizeof(__u64) > end) { ++ if (cookie + sizeof(__u64) > end) { + goto failed; + } + +- key = ngx_quic_parse_uint64(dcid); ++ key = ngx_quic_parse_uint64(cookie); + + rc = bpf_sk_select_reuseport(ctx, &ngx_quic_sockmap, &key, 0); + +diff --git a/src/event/quic/ngx_event_quic_bpf_code.c b/src/event/quic/ngx_event_quic_bpf_code.c +index 5c9dea1..1124c04 100644 +--- a/src/event/quic/ngx_event_quic_bpf_code.c ++++ b/src/event/quic/ngx_event_quic_bpf_code.c +@@ -7,62 +7,69 @@ + + + static ngx_bpf_reloc_t bpf_reloc_prog_ngx_quic_reuseport_helper[] = { +- { "ngx_quic_sockmap", 55 }, ++ { "ngx_quic_sockmap", 62 }, + }; + + static struct bpf_insn bpf_insn_prog_ngx_quic_reuseport_helper[] = { + /* opcode dst src offset imm */ + { 0x79, BPF_REG_4, BPF_REG_1, (int16_t) 0, 0x0 }, + { 0x79, BPF_REG_3, BPF_REG_1, (int16_t) 8, 0x0 }, ++ { 0xbf, BPF_REG_6, BPF_REG_4, (int16_t) 0, 0x0 }, ++ { 0x7, BPF_REG_6, BPF_REG_0, (int16_t) 0, 0x8 }, ++ { 0x2d, BPF_REG_6, BPF_REG_3, (int16_t) 61, 0x0 }, + { 0xbf, BPF_REG_2, BPF_REG_4, (int16_t) 0, 0x0 }, +- { 0x7, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0x8 }, +- { 0x2d, BPF_REG_2, BPF_REG_3, (int16_t) 54, 0x0 }, +- { 0xbf, BPF_REG_5, BPF_REG_4, (int16_t) 0, 0x0 }, +- { 0x7, BPF_REG_5, BPF_REG_0, (int16_t) 0, 0x9 }, +- { 0x2d, BPF_REG_5, BPF_REG_3, (int16_t) 51, 0x0 }, +- { 0xb7, BPF_REG_5, BPF_REG_0, (int16_t) 0, 0x14 }, ++ { 0x7, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0x9 }, ++ { 0x2d, BPF_REG_2, BPF_REG_3, (int16_t) 58, 0x0 }, ++ { 0xb7, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0x14 }, ++ { 0xb7, BPF_REG_5, BPF_REG_0, (int16_t) 0, 0x8 }, + { 0xb7, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x9 }, +- { 0x71, BPF_REG_6, BPF_REG_2, (int16_t) 0, 0x0 }, ++ { 0x71, BPF_REG_6, BPF_REG_6, (int16_t) 0, 0x0 }, + { 0x67, BPF_REG_6, BPF_REG_0, (int16_t) 0, 0x38 }, + { 0xc7, BPF_REG_6, BPF_REG_0, (int16_t) 0, 0x38 }, +- { 0x65, BPF_REG_6, BPF_REG_0, (int16_t) 10, 0xffffffff }, ++ { 0x65, BPF_REG_6, BPF_REG_0, (int16_t) 11, 0xffffffff }, + { 0xbf, BPF_REG_2, BPF_REG_4, (int16_t) 0, 0x0 }, + { 0x7, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0xd }, +- { 0x2d, BPF_REG_2, BPF_REG_3, (int16_t) 42, 0x0 }, ++ { 0x2d, BPF_REG_2, BPF_REG_3, (int16_t) 48, 0x0 }, + { 0xbf, BPF_REG_5, BPF_REG_4, (int16_t) 0, 0x0 }, + { 0x7, BPF_REG_5, BPF_REG_0, (int16_t) 0, 0xe }, +- { 0x2d, BPF_REG_5, BPF_REG_3, (int16_t) 39, 0x0 }, ++ { 0x2d, BPF_REG_5, BPF_REG_3, (int16_t) 45, 0x0 }, ++ { 0xb7, BPF_REG_5, BPF_REG_0, (int16_t) 0, 0xd }, + { 0xb7, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0xe }, +- { 0x71, BPF_REG_5, BPF_REG_2, (int16_t) 0, 0x0 }, ++ { 0x71, BPF_REG_2, BPF_REG_2, (int16_t) 0, 0x0 }, + { 0xb7, BPF_REG_6, BPF_REG_0, (int16_t) 0, 0x8 }, +- { 0x2d, BPF_REG_6, BPF_REG_5, (int16_t) 35, 0x0 }, +- { 0xf, BPF_REG_5, BPF_REG_0, (int16_t) 0, 0x0 }, ++ { 0x2d, BPF_REG_6, BPF_REG_2, (int16_t) 40, 0x0 }, ++ { 0xbf, BPF_REG_6, BPF_REG_2, (int16_t) 0, 0x0 }, ++ { 0xf, BPF_REG_6, BPF_REG_0, (int16_t) 0, 0x0 }, ++ { 0xbf, BPF_REG_0, BPF_REG_4, (int16_t) 0, 0x0 }, ++ { 0xf, BPF_REG_0, BPF_REG_6, (int16_t) 0, 0x0 }, ++ { 0x2d, BPF_REG_0, BPF_REG_3, (int16_t) 35, 0x0 }, + { 0xf, BPF_REG_4, BPF_REG_5, (int16_t) 0, 0x0 }, +- { 0x2d, BPF_REG_4, BPF_REG_3, (int16_t) 32, 0x0 }, ++ { 0xf, BPF_REG_2, BPF_REG_4, (int16_t) 0, 0x0 }, + { 0xbf, BPF_REG_4, BPF_REG_2, (int16_t) 0, 0x0 }, +- { 0x7, BPF_REG_4, BPF_REG_0, (int16_t) 0, 0x9 }, +- { 0x2d, BPF_REG_4, BPF_REG_3, (int16_t) 29, 0x0 }, +- { 0x71, BPF_REG_4, BPF_REG_2, (int16_t) 1, 0x0 }, ++ { 0x7, BPF_REG_4, BPF_REG_0, (int16_t) 0, 0x1 }, ++ { 0x2d, BPF_REG_4, BPF_REG_3, (int16_t) 30, 0x0 }, ++ { 0x7, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0xfffffff9 }, ++ { 0x71, BPF_REG_4, BPF_REG_2, (int16_t) 0, 0x0 }, + { 0x67, BPF_REG_4, BPF_REG_0, (int16_t) 0, 0x38 }, +- { 0x71, BPF_REG_3, BPF_REG_2, (int16_t) 2, 0x0 }, ++ { 0x71, BPF_REG_3, BPF_REG_2, (int16_t) 1, 0x0 }, + { 0x67, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0x30 }, + { 0x4f, BPF_REG_3, BPF_REG_4, (int16_t) 0, 0x0 }, +- { 0x71, BPF_REG_4, BPF_REG_2, (int16_t) 3, 0x0 }, ++ { 0x71, BPF_REG_4, BPF_REG_2, (int16_t) 2, 0x0 }, + { 0x67, BPF_REG_4, BPF_REG_0, (int16_t) 0, 0x28 }, + { 0x4f, BPF_REG_3, BPF_REG_4, (int16_t) 0, 0x0 }, +- { 0x71, BPF_REG_4, BPF_REG_2, (int16_t) 4, 0x0 }, ++ { 0x71, BPF_REG_4, BPF_REG_2, (int16_t) 3, 0x0 }, + { 0x67, BPF_REG_4, BPF_REG_0, (int16_t) 0, 0x20 }, + { 0x4f, BPF_REG_3, BPF_REG_4, (int16_t) 0, 0x0 }, +- { 0x71, BPF_REG_4, BPF_REG_2, (int16_t) 5, 0x0 }, ++ { 0x71, BPF_REG_4, BPF_REG_2, (int16_t) 4, 0x0 }, + { 0x67, BPF_REG_4, BPF_REG_0, (int16_t) 0, 0x18 }, + { 0x4f, BPF_REG_3, BPF_REG_4, (int16_t) 0, 0x0 }, +- { 0x71, BPF_REG_4, BPF_REG_2, (int16_t) 6, 0x0 }, ++ { 0x71, BPF_REG_4, BPF_REG_2, (int16_t) 5, 0x0 }, + { 0x67, BPF_REG_4, BPF_REG_0, (int16_t) 0, 0x10 }, + { 0x4f, BPF_REG_3, BPF_REG_4, (int16_t) 0, 0x0 }, +- { 0x71, BPF_REG_4, BPF_REG_2, (int16_t) 7, 0x0 }, ++ { 0x71, BPF_REG_4, BPF_REG_2, (int16_t) 6, 0x0 }, + { 0x67, BPF_REG_4, BPF_REG_0, (int16_t) 0, 0x8 }, + { 0x4f, BPF_REG_3, BPF_REG_4, (int16_t) 0, 0x0 }, +- { 0x71, BPF_REG_2, BPF_REG_2, (int16_t) 8, 0x0 }, ++ { 0x71, BPF_REG_2, BPF_REG_2, (int16_t) 7, 0x0 }, + { 0x4f, BPF_REG_3, BPF_REG_2, (int16_t) 0, 0x0 }, + { 0x7b, BPF_REG_10, BPF_REG_3, (int16_t) 65528, 0x0 }, + { 0xbf, BPF_REG_3, BPF_REG_10, (int16_t) 0, 0x0 }, +@@ -86,3 +93,4 @@ ngx_bpf_program_t ngx_quic_reuseport_helper = { + .license = "BSD", + .type = BPF_PROG_TYPE_SK_REUSEPORT, + }; ++ +diff --git a/src/event/quic/ngx_event_quic_connid.c b/src/event/quic/ngx_event_quic_connid.c +index f508682..9046db1 100644 +--- a/src/event/quic/ngx_event_quic_connid.c ++++ b/src/event/quic/ngx_event_quic_connid.c +@@ -15,6 +15,8 @@ + #if (NGX_QUIC_BPF) + static ngx_int_t ngx_quic_bpf_attach_id(ngx_connection_t *c, u_char *id); + #endif ++static ngx_int_t ngx_quic_dcid_encode_server_info(ngx_connection_t *c, ++ u_char *id); + static ngx_int_t ngx_quic_retire_client_id(ngx_connection_t *c, + ngx_quic_client_id_t *cid); + static ngx_quic_client_id_t *ngx_quic_alloc_client_id(ngx_connection_t *c, +@@ -38,6 +40,12 @@ ngx_quic_create_server_id(ngx_connection_t *c, u_char *id) + } + #endif + ++ /* encode server info into DCID for L4LB/DPVS */ ++ if (ngx_quic_dcid_encode_server_info(c, id) != NGX_OK) { ++ ngx_log_error(NGX_LOG_ERR, c->log, 0, ++ "quic server info failed to be encoded"); ++ } ++ + return NGX_OK; + } + +@@ -69,6 +77,60 @@ ngx_quic_bpf_attach_id(ngx_connection_t *c, u_char *id) + + #endif + ++/* ++ * L4LB/DPVS QUIC Connction ID Format { ++ * First Octet (8), ++ * L3 Address Length (3), ++ * L4 Address Flag (1), ++ * L3 Address (8...64), ++ * [ L4 Address (16) ] ++ * Nonce (32...140) ++ * } ++ * ++ * Specifically for this case: ++ * L3 Address Length := 3 (IPv4), 4 (IPv6) ++ * L4 Address Flag := 0 ++ */ ++static ngx_int_t ++ngx_quic_dcid_encode_server_info(ngx_connection_t *c, u_char *id) ++{ ++ unsigned int len; ++ u_char *addr, *ptr; ++ struct sockaddr *sa; ++ ++ sa = c->local_sockaddr; ++ if (sa->sa_family == AF_INET) { ++ addr = (u_char *)(&((struct sockaddr_in *)sa)->sin_addr); ++ len = 3; ++ addr += (4 - len); ++ } else if (sa->sa_family == AF_INET6) { ++ addr = (u_char *)(&((struct sockaddr_in6 *)sa)->sin6_addr); ++ len = 4; ++ addr += (16 - len); ++ } else { ++ return NGX_OK; ++ } ++ ++ if (len + sizeof(uint64_t) + 2 > NGX_QUIC_SERVER_CID_LEN) { ++ return NGX_ERROR; ++ } ++ ++ ptr = id; ++ ptr++; ++ ++ *ptr = 0; ++ *ptr++ = (((len - 1) & 0x7) << 5) | ((*addr >> 4) & 0xf); ++ ++ while (--len > 0) { ++ *ptr++ = ((*addr & 0xf) << 4) | ((*(addr+1) >> 4) & 0xf); ++ addr++; ++ } ++ ++ *ptr &= 0xf; ++ *ptr |= ((*addr & 0xf) << 4); ++ ++ return NGX_OK; ++} + + ngx_int_t + ngx_quic_handle_new_connection_id_frame(ngx_connection_t *c, +diff --git a/src/event/quic/ngx_event_quic_transport.c b/src/event/quic/ngx_event_quic_transport.c +index 19670a6..1cbec6c 100644 +--- a/src/event/quic/ngx_event_quic_transport.c ++++ b/src/event/quic/ngx_event_quic_transport.c +@@ -2198,5 +2198,6 @@ ngx_quic_create_close(u_char *p, ngx_quic_frame_t *f) + void + ngx_quic_dcid_encode_key(u_char *dcid, uint64_t key) + { ++ dcid += (NGX_QUIC_SERVER_CID_LEN - sizeof(key)); + (void) ngx_quic_write_uint64(dcid, key); + } +-- +1.8.3.1 +