假冒建设厅网站/优化加速
测试拓扑
双臂模式, 使用两个网卡,一个对外,一个对内。
Client host是物理机, RS host都是虚拟机。
LB host是物理机,两个CX5网卡分别在两个子网。
配置文件
用dpvs.conf.sample作为双臂配置文件,其中多一个接口配置,以及worker中多一个接口tx/rx队列
netif_defs {<init> pktpool_size 1048575<init> pktpool_cache 256<init> fdir_mode perfect# dpdk0 dpdk1 两个接口<init> device dpdk0 {rx {queue_number 8descriptor_number 1024rss all}tx {queue_number 8descriptor_number 1024}! mtu 1500! promisc_mode! allmulticastkni_name dpdk0.kni}<init> device dpdk1 {rx {queue_number 8descriptor_number 1024rss all}tx {queue_number 8descriptor_number 1024}! mtu 1500! promisc_mode! allmulticastkni_name dpdk1.kni}
}worker_defs {<init> worker cpu0 {type mastercpu_id 0}# worker中也增加了一个接口tx/rx队列<init> worker cpu1 {type slavecpu_id 1port dpdk0 {rx_queue_ids 0tx_queue_ids 0! isol_rx_cpu_ids 9! isol_rxq_ring_sz 1048576}port dpdk1 {rx_queue_ids 0tx_queue_ids 0! isol_rx_cpu_ids 9! isol_rxq_ring_sz 1048576}}
启动DPVS
命令行中 -a 98:00.0 -a 98:00.1 设置使用的两张网卡。
./dpvs -c ../conf/dpvs.conf.sample -- -a 98:00.0 -a 98:00.1 -l 0-9root@r750-132:~/dpvs/bin# ./dpvs -c ../conf/dpvs.conf.sample -- -a 98:00.0 -a 98:00.1 -l 0-8
current thread affinity is set to FFFFFFFF
EAL: Detected 32 lcore(s)
EAL: Detected 2 NUMA nodes
EAL: Detected static linkage of DPDK
EAL: Multi-process socket /var/run/dpdk/rte/mp_socket
EAL: Selected IOVA mode 'PA'
EAL: No available hugepages reported in hugepages-1048576kB
EAL: Probing VFIO support...
EAL: VFIO support initialized
EAL: Probe PCI driver: mlx5_pci (15b3:1017) device: 0000:98:00.0 (socket 1)
EAL: Probe PCI driver: mlx5_pci (15b3:1017) device: 0000:98:00.1 (socket 1)
EAL: No legacy callbacks, legacy socket not created
DPVS: dpvs version: 1.9-8, build on 2025.02.20.15:37:58
DPVS: dpvs-conf-file: ../conf/dpvs.conf.sample
DPVS: dpvs-pid-file: /var/run/dpvs.pid
DPVS: dpvs-ipc-file: /var/run/dpvs.ipc
CFG_FILE: Opening configuration file '../conf/dpvs.conf.sample'.
CFG_FILE: log_level = WARNING
NETIF: dpdk0:rx_queue_number = 8
NETIF: dpdk1:rx_queue_number = 8
NETIF: worker cpu1:dpdk0 rx_queue_id += 0
NETIF: worker cpu1:dpdk0 tx_queue_id += 0
NETIF: worker cpu1:dpdk1 rx_queue_id += 0
NETIF: worker cpu1:dpdk1 tx_queue_id += 0
NETIF: worker cpu2:dpdk0 rx_queue_id += 1
NETIF: worker cpu2:dpdk0 tx_queue_id += 1
NETIF: worker cpu2:dpdk1 rx_queue_id += 1
NETIF: worker cpu2:dpdk1 tx_queue_id += 1
NETIF: worker cpu3:dpdk0 rx_queue_id += 2
NETIF: worker cpu3:dpdk0 tx_queue_id += 2
NETIF: worker cpu3:dpdk1 rx_queue_id += 2
NETIF: worker cpu3:dpdk1 tx_queue_id += 2
NETIF: worker cpu4:dpdk0 rx_queue_id += 3
NETIF: worker cpu4:dpdk0 tx_queue_id += 3
NETIF: worker cpu4:dpdk1 rx_queue_id += 3
NETIF: worker cpu4:dpdk1 tx_queue_id += 3
NETIF: worker cpu5:dpdk0 rx_queue_id += 4
NETIF: worker cpu5:dpdk0 tx_queue_id += 4
NETIF: worker cpu5:dpdk1 rx_queue_id += 4
NETIF: worker cpu5:dpdk1 tx_queue_id += 4
NETIF: worker cpu6:dpdk0 rx_queue_id += 5
NETIF: worker cpu6:dpdk0 tx_queue_id += 5
NETIF: worker cpu6:dpdk1 rx_queue_id += 5
NETIF: worker cpu6:dpdk1 tx_queue_id += 5
NETIF: worker cpu7:dpdk0 rx_queue_id += 6
NETIF: worker cpu7:dpdk0 tx_queue_id += 6
NETIF: worker cpu7:dpdk1 rx_queue_id += 6
NETIF: worker cpu7:dpdk1 tx_queue_id += 6
NETIF: worker cpu8:dpdk0 rx_queue_id += 7
NETIF: worker cpu8:dpdk0 tx_queue_id += 7
NETIF: worker cpu8:dpdk1 rx_queue_id += 7
NETIF: worker cpu8:dpdk1 tx_queue_id += 7
SAPOOL: sapool_filter_enable = on
NETIF: Ethdev port_id=0 invalid tx_offload: 0x1000e, valid value: 0xc96af
NETIF: Ethdev port_id=1 invalid tx_offload: 0x1000e, valid value: 0xc96af
负载均衡配置
# 添加VIP
./dpip addr add 10.0.0.100/32 dev dpdk0# 添加负载均衡服务 , 轮询模式
./ipvsadm -A -t 10.0.0.100:80 -s rr# 添加 3个RS, FULLNAT 模式
./ipvsadm -a -t 10.0.0.100:80 -r 192.168.100.3:80 -b
./ipvsadm -a -t 10.0.0.100:80 -r 192.168.100.4:80 -b
./ipvsadm -a -t 10.0.0.100:80 -r 192.168.100.5:80 -b# 为负载均衡服务 10.0.0.100:80 添加一个LOCAL IP 在dpdk1上
./ipvsadm --add-laddr -z 192.168.100.200 -t 10.0.0.100:80 -F dpdk1# 添加路由
./dpip route add 10.0.0.0/16 dev dpdk0
./dpip route add 192.168.100.0/24 dev dpdk1
检查配置
服务配置
root@r750-132:~/dpvs/bin# ./ipvsadm -L -n
IP Virtual Server version 1.9.8 (size=0)
Prot LocalAddress:Port Scheduler Flags-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 10.0.0.100:80 rr-> 192.168.100.3:0 FullNat 1 0 0 -> 192.168.100.4:0 FullNat 1 0 0 -> 192.168.100.5:0 FullNat 1 0 0
root@r750-132:~/dpvs/bin# ./ipvsadm -G
VIP:VPORT TOTAL SNAT_IP CONFLICTS CONNS
10.0.0.100:80 1 192.168.100.200 0 0
接口路由配置
root@r750-132:~/dpvs/bin# ./dpip addr show
inet 10.0.0.100/32 scope global dpdk0valid_lft forever preferred_lft forever
inet6 fe80::eaeb:d3ff:fea3:8377/64 scope link dpdk1valid_lft forever preferred_lft forever
inet 192.168.100.200/32 scope global dpdk1valid_lft forever preferred_lft forever
inet6 fe80::eaeb:d3ff:fea3:8376/64 scope link dpdk0valid_lft forever preferred_lft forever
root@r750-132:~/dpvs/bin#
root@r750-132:~/dpvs/bin# ./dpip route show
inet 10.0.0.100/32 via 0.0.0.0 src 0.0.0.0 dev dpdk0 mtu 1500 tos 0 scope host metric 0 proto auto
inet 192.168.100.200/32 via 0.0.0.0 src 0.0.0.0 dev dpdk1 mtu 1500 tos 0 scope host metric 0 proto auto
inet 192.168.100.0/24 via 0.0.0.0 src 0.0.0.0 dev dpdk1 mtu 1500 tos 0 scope link metric 0 proto auto
inet 10.0.0.0/16 via 0.0.0.0 src 0.0.0.0 dev dpdk0 mtu 1500 tos 0 scope link metric 0 proto auto
检查连通性
Client ping dpdk0
# Client ping dpdk0
[root@dkdp192 ~]# ifconfig enp1s0f0np0 10.0.0.192/24
[root@dkdp192 ~]#
[root@dkdp192 ~]# ping 10.0.0.100
PING 10.0.0.100 (10.0.0.100) 56(84) bytes of data.
64 bytes from 10.0.0.100: icmp_seq=1 ttl=64 time=0.153 ms
64 bytes from 10.0.0.100: icmp_seq=2 ttl=64 time=0.068 ms
64 bytes from 10.0.0.100: icmp_seq=3 ttl=64 time=0.080 ms
^C
--- 10.0.0.100 ping statistics ---
3 packets transmitted, 3 received, 0% packet loss, time 2063ms
rtt min/avg/max/mdev = 0.068/0.100/0.153/0.038 ms
RS ping dpdk1
root@ubuntu22:~# ping 192.168.100.200
PING 192.168.100.200 (192.168.100.200) 56(84) bytes of data.
64 bytes from 192.168.100.200: icmp_seq=1 ttl=64 time=0.328 ms
64 bytes from 192.168.100.200: icmp_seq=2 ttl=64 time=0.118 ms
64 bytes from 192.168.100.200: icmp_seq=3 ttl=64 time=0.085 ms
^C
--- 192.168.100.200 ping statistics ---
3 packets transmitted, 3 received, 0% packet loss, time 2042ms
rtt min/avg/max/mdev = 0.085/0.177/0.328/0.107 ms
CURL测试
测试成功
[root@dkdp192 ~]# curl 10.0.0.100:80
This is Server 2 !
[root@dkdp192 ~]# curl 10.0.0.100:80
This is Server 1 !
[root@dkdp192 ~]# curl 10.0.0.100:80
This is Server 2 !
[root@dkdp192 ~]# curl 10.0.0.100:80
This is Server 0 !
[root@dkdp192 ~]# curl 10.0.0.100:80
This is Server 0 !
[root@dkdp192 ~]# curl 10.0.0.100:80
This is Server 0 !
[root@dkdp192 ~]# curl 10.0.0.100:80
This is Server 1 !
[root@dkdp192 ~]# curl 10.0.0.100:80
This is Server 2 !
[root@dkdp192 ~]# curl 10.0.0.100:80
This is Server 1 !
[root@dkdp192 ~]# curl 10.0.0.100:80
This is Server 0 !
[root@dkdp192 ~]# curl 10.0.0.100:80
This is Server 1 !
[root@dkdp192 ~]# curl 10.0.0.100:80
This is Server 2 !
[root@dkdp192 ~]# curl 10.0.0.100:80
This is Server 2 !
[root@dkdp192 ~]# curl 10.0.0.100:80
This is Server 2 !
[root@dkdp192 ~]# curl 10.0.0.100:80
This is Server 0 !
[root@dkdp192 ~]# curl 10.0.0.100:80
This is Server 2 !
[root@dkdp192 ~]# curl 10.0.0.100:80
This is Server 1 !
[root@dkdp192 ~]# curl 10.0.0.100:80
This is Server 1 !
[root@dkdp192 ~]#