首先启动两个tcp server, 代码里开启 SO_REUSEPORT
[my_test@localhost test]$ ./tcp_server_reuseport &
[1] 1864
[my_test@localhost test]$ Server listening on port 8888
[my_test@localhost test]$ ./tcp_server_reuseport &
[2] 1865
[my_test@localhost test]$ Server listening on port 8888
[my_test@localhost test]$ ps -ef|grep tcp_server_reuseport
my_test 1864 1443 0 23:11 pts/0 00:00:00 ./tcp_server_reuseport
my_test 1865 1443 0 23:11 pts/0 00:00:00 ./tcp_server_reuseport
启动10条客户端连接:
[my_test@localhost test]$ ./tcp_client_reuseport
Message sent
Server response: Welcome to the server! Server PID: 1865
Message sent
Server response: Welcome to the server! Server PID: 1864
Message sent
Server response: Welcome to the server! Server PID: 1865
Message sent
Server response: Welcome to the server! Server PID: 1864
Message sent
Server response: Welcome to the server! Server PID: 1864
Message sent
Server response: Welcome to the server! Server PID: 1865
Message sent
Server response: Welcome to the server! Server PID: 1865
Message sent
Server response: Welcome to the server! Server PID: 1865
Message sent
Server response: Welcome to the server! Server PID: 1864
Message sent
Server response: Welcome to the server! Server PID: 1864
从回复的server 进程id 可见,负载均衡做的很好。
追踪内核调用链(内核版本5.10.216 x86_64):
在server端收到client的SYN连接请求时会触发 __inet_lookup_listener -> inet_lhash2_lookup -> lookup_reuseport -> reuseport_select_sock 通过哈希选择一个Listen Socket来处理这个连接请求。
(gdb) b reuseport_select_sock
Breakpoint 1 at 0xffffffff8199ac80: file net/core/sock_reuseport.c, line 277.
(gdb) c
Continuing.
[New Thread 2061]
[Switching to Thread 2061]
Thread 293 hit Breakpoint 1, reuseport_select_sock (sk=sk@entry=0xffff888038c90000, hash=317205834, skb=skb@entry=0xffff88800a90dce0,
hdr_len=hdr_len@entry=40) at net/core/sock_reuseport.c:277
277 net/core/sock_reuseport.c: No such file or directory.
(gdb) bt
#0 reuseport_select_sock (sk=sk@entry=0xffff888038c90000, hash=317205834, skb=skb@entry=0xffff88800a90dce0, hdr_len=hdr_len@entry=40)
at net/core/sock_reuseport.c:277
#1 0xffffffff81a03ff8 in lookup_reuseport (hnum=8888, daddr=0, sport=21715, saddr=251789322, doff=40, skb=0xffff88800a90dce0,
sk=0xffff888038c90000, net=0xffffffff82a1ac40 <init_net>) at net/ipv4/inet_hashtables.c:265
#2 lookup_reuseport (hnum=8888, daddr=0, sport=21715, saddr=251789322, doff=40, skb=0xffff88800a90dce0, sk=0xffff888038c90000,
net=0xffffffff82a1ac40 <init_net>) at net/ipv4/inet_hashtables.c:255
#3 inet_lhash2_lookup (net=net@entry=0xffffffff82a1ac40 <init_net>, ilb2=<optimized out>, skb=skb@entry=0xffff88800a90dce0,
doff=doff@entry=40, saddr=saddr@entry=251789322, sport=sport@entry=21715, daddr=0, hnum=8888, dif=2, sdif=0)
at net/ipv4/inet_hashtables.c:293
#4 0xffffffff81a042f3 in __inet_lookup_listener (net=net@entry=0xffffffff82a1ac40 <init_net>,
hashinfo=hashinfo@entry=0xffffffff832a69c0 <tcp_hashinfo>, skb=skb@entry=0xffff88800a90dce0, doff=doff@entry=40,
saddr=saddr@entry=251789322, sport=sport@entry=21715, daddr=<optimized out>, hnum=8888, dif=2, sdif=0)
at net/ipv4/inet_hashtables.c:361
#5 0xffffffff81a25978 in __inet_lookup (hashinfo=0xffffffff832a69c0 <tcp_hashinfo>, sdif=0, refcounted=<synthetic pointer>, dif=2,
dport=<optimized out>, daddr=251789322, sport=<optimized out>, saddr=251789322, doff=<optimized out>, skb=0xffff88800a90dce0,
net=0xffffffff82a1ac40 <init_net>) at ./include/net/inet_hashtables.h:343
#6 __inet_lookup_skb (hashinfo=0xffffffff832a69c0 <tcp_hashinfo>, sdif=0, refcounted=<synthetic pointer>, dport=<optimized out>,
sport=<optimized out>, doff=<optimized out>, skb=0xffff88800a90dce0) at ./include/net/inet_hashtables.h:379
#7 tcp_v4_rcv (skb=0xffff88800a90dce0) at net/ipv4/tcp_ipv4.c:1984
#8 0xffffffff819f922b in ip_protocol_deliver_rcu (net=0xffffffff82a1ac40 <init_net>, skb=0xffff88800a90dce0,
protocol=<optimized out>) at net/ipv4/ip_input.c:204
#9 0xffffffff819f93ef in ip_local_deliver_finish (net=<optimized out>, sk=<optimized out>, skb=<optimized out>)
at ./include/linux/skbuff.h:2533
#10 0xffffffff819f94fa in NF_HOOK (sk=0x0 <fixed_percpu_data>, pf=2 '\002', hook=1, in=<optimized out>, out=0x0 <fixed_percpu_data>,
okfn=0xffffffff819f93b0 <ip_local_deliver_finish>, skb=0xffff88800a90dce0, net=0xffffffff82a1ac40 <init_net>)
at ./include/linux/netfilter.h:296
#11 NF_HOOK (pf=2 '\002', sk=0x0 <fixed_percpu_data>, out=0x0 <fixed_percpu_data>, okfn=0xffffffff819f93b0 <ip_local_deliver_finish>,
in=<optimized out>, skb=0xffff88800a90dce0, net=0xffffffff82a1ac40 <init_net>, hook=1) at ./include/linux/netfilter.h:290
#12 ip_local_deliver (skb=0xffff88800a90dce0) at net/ipv4/ip_input.c:252
#13 0xffffffff819f95e3 in NF_HOOK (sk=0x0 <fixed_percpu_data>, pf=2 '\002', hook=0, in=0xffff88800391b000,
out=0x0 <fixed_percpu_data>, okfn=0xffffffff819f8c40 <ip_rcv_finish>, skb=0xffff88800a90dce0, net=0xffffffff82a1ac40 <init_net>)
at ./include/linux/netfilter.h:296
#14 NF_HOOK (pf=2 '\002', sk=0x0 <fixed_percpu_data>, out=0x0 <fixed_percpu_data>, okfn=0xffffffff819f8c40 <ip_rcv_finish>,
--Type <RET> for more, q to quit, c to continue without paging--
in=0xffff88800391b000, skb=0xffff88800a90dce0, net=0xffffffff82a1ac40 <init_net>, hook=0) at ./include/linux/netfilter.h:290
#15 ip_rcv (skb=0xffff88800a90dce0, dev=0xffff88800391b000, pt=<optimized out>, orig_dev=<optimized out>) at net/ipv4/ip_input.c:551
#16 0xffffffff819714a4 in __netif_receive_skb_one_core (skb=<optimized out>, pfmemalloc=<optimized out>) at net/core/dev.c:5375
#17 0xffffffff819716e9 in process_backlog (napi=0xffff88807dc2e3d0, quota=64) at net/core/dev.c:6396
#18 0xffffffff81972fce in napi_poll (repoll=0xffffc90000003f60, n=0xffff88807dc2e3d0) at net/core/dev.c:6847
#19 net_rx_action (h=<optimized out>) at net/core/dev.c:6917
#20 0xffffffff820000b7 in __do_softirq () at kernel/softirq.c:298
#21 0xffffffff81e0106f in asm_call_on_stack () at arch/x86/entry/entry_64.S:801
#22 0xffffffff81022372 in __run_on_irqstack (func=<optimized out>) at ./arch/x86/include/asm/irq_stack.h:26
#23 run_on_irqstack_cond (regs=0x0 <fixed_percpu_data>, func=<optimized out>) at ./arch/x86/include/asm/irq_stack.h:77
#24 do_softirq_own_stack () at arch/x86/kernel/irq_64.c:77
#25 0xffffffff8106c7aa in do_softirq () at kernel/softirq.c:343
#26 0xffffffff8106c7fa in do_softirq () at ./arch/x86/include/asm/preempt.h:26
#27 __local_bh_enable_ip (ip=ip@entry=18446744071589316362, cnt=cnt@entry=512) at kernel/softirq.c:195
#28 0xffffffff819fc71b in local_bh_enable () at ./include/linux/bottom_half.h:32
#29 rcu_read_unlock_bh () at ./include/linux/rcupdate.h:806
#30 ip_finish_output2 (net=<optimized out>, sk=<optimized out>, skb=<optimized out>) at net/ipv4/ip_output.c:238
#31 0xffffffff819feee8 in ip_finish_output (skb=0xffff88800a90dce0, sk=0xffff888038c92bc0, net=0xffffffff82a1ac40 <init_net>)
at net/ipv4/ip_output.c:325
#32 NF_HOOK_COND (pf=2 '\002', hook=4, okfn=0xffffffff819fd670 <ip_finish_output>, cond=<optimized out>, out=<optimized out>,
in=<optimized out>, skb=0xffff88800a90dce0, sk=0xffff888038c92bc0, net=0xffffffff82a1ac40 <init_net>)
at ./include/linux/netfilter.h:285
#33 ip_output (net=0xffffffff82a1ac40 <init_net>, sk=0xffff888038c92bc0, skb=0xffff88800a90dce0) at net/ipv4/ip_output.c:439
#34 0xffffffff819fe982 in __ip_queue_xmit (sk=0xffff888038c92bc0, skb=0xffff88800a90dce0, fl=0xffff888038c92f20, tos=<optimized out>)
at net/ipv4/ip_output.c:540
#35 0xffffffff819febbc in ip_queue_xmit (sk=<optimized out>, skb=<optimized out>, fl=<optimized out>) at ./include/net/inet_sock.h:302
#36 0xffffffff81a1bef6 in __tcp_transmit_skb (sk=sk@entry=0xffff888038c92bc0, skb=0xffff88800a90dce0, skb@entry=0xffff88800a90dc00,
clone_it=clone_it@entry=1, gfp_mask=<optimized out>, rcv_nxt=<optimized out>) at net/ipv4/tcp_output.c:1407
#37 0xffffffff81a1c8ed in tcp_transmit_skb (gfp_mask=<optimized out>, clone_it=1, skb=0xffff88800a90dc00, sk=0xffff888038c92bc0)
at ./include/linux/tcp.h:439
#38 tcp_connect (sk=sk@entry=0xffff888038c92bc0) at net/ipv4/tcp_output.c:3888
#39 0xffffffff81a22c90 in tcp_v4_connect (sk=0xffff888038c92bc0, uaddr=0xffffc90000d43e88, addr_len=<optimized out>)
at net/ipv4/tcp_ipv4.c:314
--Type <RET> for more, q to quit, c to continue without paging--
#40 0xffffffff81a3d87c in __inet_stream_connect (sock=sock@entry=0xffff88803652f000, uaddr=0x12e82d4a,
uaddr@entry=0xffffc90000d43e88, addr_len=177265888, addr_len@entry=16, flags=40, flags@entry=2, is_sendmsg=21715,
is_sendmsg@entry=0) at net/ipv4/af_inet.c:666
#41 0xffffffff81a3db41 in inet_stream_connect (sock=0xffff88803652f000, uaddr=0xffffc90000d43e88, addr_len=16, flags=2)
at net/ipv4/af_inet.c:730
#42 0xffffffff8194bdc5 in __sys_connect (fd=<optimized out>, uservaddr=0x7ffe8421f910, addrlen=16) at net/socket.c:1882
#43 0xffffffff8194be01 in __do_sys_connect (addrlen=<optimized out>, uservaddr=<optimized out>, fd=<optimized out>)
at net/socket.c:1892
#44 __se_sys_connect (addrlen=<optimized out>, uservaddr=<optimized out>, fd=<optimized out>) at net/socket.c:1889
#45 __x64_sys_connect (regs=<optimized out>) at net/socket.c:1889
#46 0xffffffff81c38090 in do_syscall_64 (nr=<optimized out>, regs=0xffffc90000d43f58) at arch/x86/entry/common.c:46
#47 0xffffffff81e0011f in entry_SYSCALL_64 () at arch/x86/entry/entry_64.S:125
#48 0x0000000000000000 in ?? ()
具体代码如下:
server端演示代码:
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <arpa/inet.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <sys/types.h>
#define PORT 8888
#define MAX_PENDING_CONNECTIONS 10
int main() {
int server_fd, new_socket;
struct sockaddr_in address;
int opt = 1;
int addrlen = sizeof(address);
// 创建套接字
if ((server_fd = socket(AF_INET, SOCK_STREAM, 0)) == 0) {
perror("socket failed");
exit(EXIT_FAILURE);
}
// 设置套接字选项
if (setsockopt(server_fd, SOL_SOCKET, SO_REUSEPORT, &opt, sizeof(opt))) {
perror("setsockopt");
exit(EXIT_FAILURE);
}
address.sin_family = AF_INET;
address.sin_addr.s_addr = INADDR_ANY;
address.sin_port = htons(PORT);
// 将套接字绑定到指定的地址和端口
if (bind(server_fd, (struct sockaddr *)&address, sizeof(address))<0) {
perror("bind failed");
exit(EXIT_FAILURE);
}
// 开始监听连接
if (listen(server_fd, MAX_PENDING_CONNECTIONS) < 0) {
perror("listen");
exit(EXIT_FAILURE);
}
printf("Server listening on port %d\n", PORT);
while(1) {
// 接受新连接
if ((new_socket = accept(server_fd, (struct sockaddr *)&address, (socklen_t*)&addrlen))<0) {
perror("accept");
exit(EXIT_FAILURE);
}
printf("New connection from %s:%d\n", inet_ntoa(address.sin_addr), ntohs(address.sin_port));
// 向客户端发送消息,包含当前进程ID
char welcome_message[100];
int pid = getpid(); // 获取当前进程的 ID
sprintf(welcome_message, "Welcome to the server! Server PID: %d\n", pid);
send(new_socket, welcome_message, strlen(welcome_message), 0);
// 关闭与客户端的连接
close(new_socket);
}
return 0;
}
客户端演示代码:
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <arpa/inet.h>
#include <sys/socket.h>
#define PORT 8888
#define SERVER_IP "10.0.2.15" // 服务器 IP 地址
#define MESSAGE "Hello from client"
int main() {
int sock = 0;
struct sockaddr_in serv_addr;
char message[1024] = {0};
char buffer[1024] = {0};
int i = 0;
while (i++ < 10) {
// 创建套接字
if ((sock = socket(AF_INET, SOCK_STREAM, 0)) < 0) {
perror("socket creation error");
exit(EXIT_FAILURE);
}
serv_addr.sin_family = AF_INET;
serv_addr.sin_port = htons(PORT);
if(inet_pton(AF_INET, SERVER_IP, &serv_addr.sin_addr) <= 0) {
perror("invalid address / address not supported");
exit(EXIT_FAILURE);
}
// 连接到服务器
if (connect(sock, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) {
perror("connection failed");
exit(EXIT_FAILURE);
}
// 向服务器发送消息
send(sock, MESSAGE, strlen(MESSAGE), 0);
printf("Message sent\n");
// 接收服务器的响应
if (recv(sock, buffer, sizeof(buffer), 0) < 0) {
perror("recv failed");
exit(EXIT_FAILURE);
}
printf("Server response: %s\n", buffer);
// 关闭套接字
close(sock);
}
return 0;
}