在安装Ubuntu 22.04.04系统的虚拟机上实现。依据fd.io官网的IKEv2文档：https://s3-docs.fd.io/vpp/22.10/usecases/ikev2/2_vpp.html。
将两个VPP实例通过IPSec隧道连接。操作系统环境：
user@localhost:~$ cat /etc/issue
Ubuntu 22.04.4 LTS \n \l
安装VPP版本23.10。
user@localhost:~/work$ curl -s https://packagecloud.io/install/repositories/fdio/release/script.deb.sh | sudo bash
Detected operating system as Ubuntu/jammy.
Installing /etc/apt/sources.list.d/fdio_release.list...done.
Importing packagecloud gpg key... Packagecloud gpg key imported to /etc/apt/keyrings/fdio_release-archive-keyring.gpg
done.
Running apt-get update... done.
The repository is setup! You can now install packages.
user@localhost:~/work$
user@localhost:~/work$ sudo apt-get install libvppinfra=23.10-release
user@localhost:~/work$ sudo apt-get install vpp=23.10-release
user@localhost:~/work$ sudo apt-get install vpp-plugin-core=23.10-release /* for create host-interface */
拓扑如下图所示，两个VPP实例分别为：vpp-initiator和vpp-responder，测试流量（ping）由clientns网络命名空间的接口veth_client发出，经过实例vpp-initiator，通过隧道到达实例vpp-responder，最后，到达serverns命名空间的接口veth_server。
vpp-responder vpp-initiator
|-----------------| |-----------------|
| | | |
| host-ifresp-|-ifresp ifinit-|-host-ifinit |
| 192.168.10.2/24 | | | | 192.168.10.1/24 |
| | |-----veth----| | |
| | IPSec Tunnel | |
| host-server | | host-client |
| 192.168.3.1/24 | | 192.168.5.1/24 |
|-------|---------| |-------|---------|
| |
server----------| |---------client
| |
|-----------------| veth veth |-----------------|
| | | | | |
| | | | | |
| veth_server----|---| |--|---veth_client |
| 192.168.3.2/24 | | 192.168.5.2/24 |
|-----------------| |-----------------|
serverns clientns
创建用到的3对veth接口，和两个网络命名空间。并且，将veth_client加入到clientns命名空间；将veth_server加入到serverns命名空间。即上图拓扑中的下半部分。
root@localhost:~# ip link add ifresp type veth peer name ifinit
root@localhost:~# ip link set dev ifresp up
root@localhost:~# ip link set dev ifinit up
root@localhost:~#
root@localhost:~# ip netns add clientns
root@localhost:~# ip netns add serverns
root@localhost:~#
root@localhost:~# ip link add veth_client type veth peer name client
root@localhost:~# ip link add veth_server type veth peer name server
root@localhost:~# ip link set dev veth_client up netns clientns
root@localhost:~# ip link set dev veth_server up netns serverns
分别为clientns和serverns中的接口veth_client和veth_server设置IP地址，以及两者互通的网关。
ip netns exec clientns \
bash -c "
ip link set dev lo up
ip addr add 192.168.5.2/24 dev veth_client
ip addr add fec5::2/16 dev veth_client
ip route add 192.168.3.0/24 via 192.168.5.1
ip route add fec3::0/16 via fec5::1
"
ip netns exec serverns \
bash -c "
ip link set dev lo up
ip addr add 192.168.3.2/24 dev veth_server
ip addr add fec3::2/16 dev veth_server
ip route add 192.168.5.0/24 via 192.168.3.1
ip route add fec5::0/16 via fec3::1
"
vpp-responder配置
首先启动VPP实例vpp-responder:
root@localhost:~# /usr/bin/vpp unix { \
cli-listen /tmp/vpp_resp.sock \
gid $(id -g) } \
api-segment { prefix vpp } \
plugins { plugin dpdk_plugin.so { disable } }
root@localhost:~#
root@localhost:~# ps aux | grep vpp
root 1675 5.0 0.3 17959392 51700 ? Ss 11:41 0:00 /usr/bin/vpp unix { cli-listen /tmp/vpp_resp.sock gid 0 } api-segment { prefix vpp } plugins { plugin dpdk_plugin.so { disable } }
进入实例vpp-responder的命令行，创建两个host-interface，分别为host-ifresp和host-server，配置相应的IP地址。新增IKEv2 profile名称pr1，预共享密钥Vpp123。隧道地址192.168.10.2，远端地址192.168.10.1。内网地址为192.168.3.0/24，远端内网地址为192.168.5.0/24。
创建IPIP隧道接口ipip0，源地址为192.168.10.2，目的地址为192.168.10.1。ipip0接口地址借用host-ifresp的地址192.168.10.2。
root@localhost:~# vppctl -s /tmp/vpp_resp.sock
vpp#
vpp# create host-interface name ifresp
host-ifresp
vpp# set interface ip addr host-ifresp 192.168.10.2/24
vpp# set interface state host-ifresp up
vpp#
vpp# create host-interface name server
host-server
vpp# set interface ip addr host-server 192.168.3.1/24
vpp# set interface state host-server up
vpp#
vpp# ikev2 profile add pr1
vpp# ikev2 profile set pr1 auth shared-key-mic string Vpp123
vpp# ikev2 profile set pr1 id local ip4-addr 192.168.10.2
vpp# ikev2 profile set pr1 id remote ip4-addr 192.168.10.1
vpp# ikev2 profile set pr1 traffic-selector local ip-range 192.168.3.0 - 192.168.3.255 port-range 0 - 65535 protocol 0
vpp# ikev2 profile set pr1 traffic-selector remote ip-range 192.168.5.0 - 192.168.5.255 port-range 0 - 65535 protocol 0
vpp#
vpp# create ipip tunnel src 192.168.10.2 dst 192.168.10.1
ipip0
vpp# ikev2 profile set pr1 tunnel ipip0
vpp# ip route add 192.168.5.0/24 via 192.168.10.1 ipip0
vpp# set interface unnumbered ipip0 use host-ifresp
如下地址所示。
vpp# show interface addr
host-ifresp (up):
L3 192.168.10.2/24
host-server (up):
L3 192.168.3.1/24
ipip0 (dn):
unnumbered, use host-ifresp
L3 192.168.10.2/24
如下显示接口的信息，注意其中的接口索引，在trace信息中会看到。
vpp# show interface
Name Idx State MTU (L3/IP4/IP6/MPLS) Counter Count
host-ifresp 1 up 9000/0/0/0 rx packets 891
rx bytes 102006
tx packets 874
tx bytes 100684
host-server 2 up 9000/0/0/0 rx packets 41
rx bytes 3142
tx packets 7
tx bytes 574
ipip0 3 up 9000/0/0/0 rx packets 17
rx bytes 2040
tx packets 16
tx bytes 2240
vpp-initiator配置
启动VPP实例vpp-initiator。
root@localhost:/home/user # /usr/bin/vpp unix { \
cli-listen /tmp/vpp_init.sock \
gid $(id -g) } \
api-segment { prefix vpp } \
plugins { plugin dpdk_plugin.so { disable } }
进入实例vpp-initiator的命令行，创建两个host-interface，分别为host-ifinit和host-client，配置相应的IP地址。新增IKEv2 profile名称pr1，预共享密钥Vpp123。隧道地址192.168.10.1，远端地址192.168.10.2。内网地址为192.168.5.0/24，远端内网地址为192.168.3.0/24。隧道的地址配置与vpp-responder实例中的地址相反。
作为隧道的发起端，额外配置了IKE和ESP的加密验证用的AEAD算法aes-gcm，DH算法modp-2048。
创建IPIP隧道接口ipip0，源地址为192.168.10.1，目的地址为192.168.10.2。ipip0接口地址借用host-ifresp的地址192.168.10.1。
root@localhost:/home/user# vppctl -s /tmp/vpp_init.sock
vpp#
vpp# create host-interface name ifinit
host-ifinit
vpp# set interface ip addr host-ifinit 192.168.10.1/24
vpp# set interface state host-ifinit up
vpp#
vpp# create host-interface name client
host-client
vpp# set interface ip addr host-client 192.168.5.1/24
vpp# set interface state host-client up
vpp#
vpp# ikev2 profile add pr1
vpp# ikev2 profile set pr1 auth shared-key-mic string Vpp123
vpp# ikev2 profile set pr1 id local ip4-addr 192.168.10.1
vpp# ikev2 profile set pr1 id remote ip4-addr 192.168.10.2
vpp#
vpp# ikev2 profile set pr1 traffic-selector remote ip-range 192.168.3.0 - 192.168.3.255 port-range 0 - 65535 protocol 0
vpp# ikev2 profile set pr1 traffic-selector local ip-range 192.168.5.0 - 192.168.5.255 port-range 0 - 65535 protocol 0
vpp#
vpp# ikev2 profile set pr1 responder host-ifinit 192.168.10.2
vpp# ikev2 profile set pr1 ike-crypto-alg aes-gcm-16 256 ike-dh modp-2048
vpp# ikev2 profile set pr1 esp-crypto-alg aes-gcm-16 256
vpp#
vpp# create ipip tunnel src 192.168.10.1 dst 192.168.10.2
ipip0
vpp# ikev2 profile set pr1 tunnel ipip0
vpp# ip route add 192.168.3.0/24 via 192.168.10.2 ipip0
vpp# set interface unnumbered ipip0 use host-ifinit
如下命令发起pr1隧道的建立。
vpp# ikev2 initiate sa-init pr1
如下接口IP地址。
vpp# show interface addr
host-client (up):
L3 192.168.5.1/24
host-ifinit (up):
L3 192.168.10.1/24
ipip0 (up):
unnumbered, use host-ifinit
L3 192.168.10.1/24
如下显示接口的信息，注意其中的接口索引，在trace信息中会看到。
vpp# show interface
Name Idx State MTU (L3/IP4/IP6/MPLS) Counter Count
host-client 2 up 9000/0/0/0 rx packets 55
rx bytes 4470
tx packets 19
tx bytes 1694
host-ifinit 1 up 9000/0/0/0 rx packets 902
rx bytes 103260
tx packets 889
tx bytes 102482
ipip0 3 up 9000/0/0/0 rx packets 16
rx bytes 1920
tx packets 17
tx bytes 2380
隧道状态
实例vpp-initiator的隧道状态。
vpp# show ipip tunnel
[0] instance 0 src 192.168.10.1 dst 192.168.10.2 table-ID 0 sw-if-idx 3 flags [none] dscp CS0
vpp#
vpp# show ipip tunnel-hash
src:192.168.10.1 dst:192.168.10.2 fib:0 transport:0 mode:0 -> 0
vpp#
vpp#
vpp# show ikev2 profile
profile pr1
auth-method shared-key-mic auth data Vpp123
local id-type ip4-addr data 192.168.10.1
remote id-type ip4-addr data 192.168.10.2
local traffic-selector addr 192.168.5.0 - 192.168.5.255 port 0 - 65535 protocol 0
remote traffic-selector addr 192.168.3.0 - 192.168.3.255 port 0 - 65535 protocol 0
protected tunnel ipip0
responder host-ifinit 192.168.10.2
ike-crypto-alg aes-gcm-16 256 ike-integ-alg none ike-dh modp-2048
esp-crypto-alg aes-gcm-16 256 esp-integ-alg none
lifetime 0 jitter 0 handover 0 maxdata 0
vpp#
vpp# show ikev2 sa
iip 192.168.10.1 ispi a71e7cbccebe474e rip 192.168.10.2 rspi a1e36ca0bb2ac94e
vpp#
vpp# show ikev2 sa details
iip 192.168.10.1 ispi a71e7cbccebe474e rip 192.168.10.2 rspi a1e36ca0bb2ac94e
encr:aes-gcm-16 prf:hmac-sha2-256 dh-group:modp-2048
state: AUTHENTICATED
nonce i:093df82d4e9cfcbe98a6af243e5aad64adcb762aa9d7a512e46994539d7d4b9d
r:d6ff14e37837a2008d777041382fe51c44193f6552c970df92f81f90c78a8ec8
SK_d 3c05386c08e1c41812842c332ae2a33bb22043f249f6a3afa6ec8809a85d5081
SK_e i:00000000: a78fd2d114c5b9c1ba99542ac3bc4927f1c43ee4ca729631cd5c376bc6ef9f2f
00000020: 7b8a8d83
r:00000000: 35ee14d93747188243f41c628f3b5263e59cd7a88a509aca1d57be26324b1c0c
00000020: 547375ba
SK_p i:96219cc2f3243efc43baf6d16b4db994ca7351f5eda0d670d8f2dc8404df29a6
r:d8fe1e353aa7e77817c416475c4bf2d431d72594ca40d0bff42a329fb9f40ae2
identifier (i) id-type ip4-addr data 192.168.10.1
identifier (r) id-type ip4-addr data 192.168.10.2
child sa 0:encr:aes-gcm-16 esn:yes
spi(i) 4e603f2e spi(r) e68aae85
SK_e i:834097fbf2f5c2084efbae42b41fe709ecb0fa522aab9736cb0242fd7b289e37
r:982b12841fd87207ab98419c8135d6430806abc44543384b262af1799fdfceec
traffic selectors (i):0 type 7 protocol_id 0 addr 192.168.5.0 - 192.168.5.255 port 0 - 65535
traffic selectors (r):0 type 7 protocol_id 0 addr 192.168.3.0 - 192.168.3.255 port 0 - 65535
Stats:
keepalives :14
rekey :0
SA init :0 (retransmit: 0)
retransmit: 0
SA auth :0
实例vpp-responder的隧道状态。
vpp# show ipip tunnel
[0] instance 0 src 192.168.10.2 dst 192.168.10.1 table-ID 0 sw-if-idx 3 flags [none] dscp CS0
vpp#
vpp# show ipip tunnel-hash
src:192.168.10.2 dst:192.168.10.1 fib:0 transport:0 mode:0 -> 0
vpp# show ikev2 profile
profile pr1
auth-method shared-key-mic auth data Vpp123
local id-type ip4-addr data 192.168.10.2
remote id-type ip4-addr data 192.168.10.1
local traffic-selector addr 192.168.3.0 - 192.168.3.255 port 0 - 65535 protocol 0
remote traffic-selector addr 192.168.5.0 - 192.168.5.255 port 0 - 65535 protocol 0
protected tunnel ipip0
lifetime 0 jitter 0 handover 0 maxdata 0
vpp#
vpp# show ikev2 sa
iip 192.168.10.1 ispi a71e7cbccebe474e rip 192.168.10.2 rspi a1e36ca0bb2ac94e
vpp#
vpp#
vpp# show ikev2 sa details
iip 192.168.10.1 ispi a71e7cbccebe474e rip 192.168.10.2 rspi a1e36ca0bb2ac94e
encr:aes-gcm-16 prf:hmac-sha2-256 dh-group:modp-2048
state: AUTHENTICATED
nonce i:093df82d4e9cfcbe98a6af243e5aad64adcb762aa9d7a512e46994539d7d4b9d
r:d6ff14e37837a2008d777041382fe51c44193f6552c970df92f81f90c78a8ec8
SK_d 3c05386c08e1c41812842c332ae2a33bb22043f249f6a3afa6ec8809a85d5081
SK_e i:00000000: a78fd2d114c5b9c1ba99542ac3bc4927f1c43ee4ca729631cd5c376bc6ef9f2f
00000020: 7b8a8d83
r:00000000: 35ee14d93747188243f41c628f3b5263e59cd7a88a509aca1d57be26324b1c0c
00000020: 547375ba
SK_p i:96219cc2f3243efc43baf6d16b4db994ca7351f5eda0d670d8f2dc8404df29a6
r:d8fe1e353aa7e77817c416475c4bf2d431d72594ca40d0bff42a329fb9f40ae2
identifier (i) id-type ip4-addr data 192.168.10.1
identifier (r) id-type ip4-addr data 192.168.10.2
child sa 0:encr:aes-gcm-16 esn:yes
spi(i) 4e603f2e spi(r) e68aae85
SK_e i:834097fbf2f5c2084efbae42b41fe709ecb0fa522aab9736cb0242fd7b289e37
r:982b12841fd87207ab98419c8135d6430806abc44543384b262af1799fdfceec
traffic selectors (i):0 type 7 protocol_id 0 addr 192.168.5.0 - 192.168.5.255 port 0 - 65535
traffic selectors (r):0 type 7 protocol_id 0 addr 192.168.3.0 - 192.168.3.255 port 0 - 65535
Stats:
keepalives :14
rekey :0
SA init :1 (retransmit: 0)
retransmit: 0
SA auth :1
连通测试
由命名空间clientns的接口veth_client，ping命名空间serverns中的接口veth_server的IP地址192.168.3.2，源地址为veth_client的接口地址192.168.5.2。
~# ip netns exec clientns ping 192.168.3.2
PING 192.168.3.2 (192.168.3.2) 56(84) bytes of data.
64 bytes from 192.168.3.2: icmp_seq=2 ttl=62 time=24.0 ms
64 bytes from 192.168.3.2: icmp_seq=3 ttl=62 time=18.9 ms
64 bytes from 192.168.3.2: icmp_seq=4 ttl=62 time=13.2 ms
64 bytes from 192.168.3.2: icmp_seq=5 ttl=62 time=24.0 ms
64 bytes from 192.168.3.2: icmp_seq=6 ttl=62 time=22.9 ms
^C
--- 192.168.3.2 ping statistics ---
6 packets transmitted, 5 received, 16.6667% packet loss, time 5030ms
rtt min/avg/max/mdev = 13.167/20.602/24.030/4.158 ms
实例vpp-initiator上的ping请求报文trace信息。
vpp# trace add af-packet-input 100
vpp#
vpp# show trace
Packet 2
00:10:11:809900: af-packet-input
af_packet: hw_if_index 2 rx-queue 0 next-index 4
block 34:
address 0x7f1245f04000 version 2 seq_num 35 pkt_num 0
tpacket3_hdr:
status 0x20000001 len 98 snaplen 98 mac 92 net 106
sec 0x661f311d nsec 0x3a5c9dd3 vlan 0 vlan_tpid 0
vnet-hdr:
flags 0x00 gso_type 0x00 hdr_len 0
gso_size 0 csum_start 0 csum_offset 0
00:10:11:809908: ethernet-input
IP4: 96:85:ba:4b:2a:90 -> 02:fe:01:74:71:41
00:10:11:809912: ip4-input
ICMP: 192.168.5.2 -> 192.168.3.2
tos 0x00, ttl 64, length 84, checksum 0x2627 dscp CS0 ecn NON_ECN
fragment id 0x8b2d, flags DONT_FRAGMENT
ICMP echo_request checksum 0xd04c id 24662
00:10:11:809915: ip4-lookup
fib 0 dpo-idx 4 flow hash: 0x00000000
ICMP: 192.168.5.2 -> 192.168.3.2
tos 0x00, ttl 64, length 84, checksum 0x2627 dscp CS0 ecn NON_ECN
fragment id 0x8b2d, flags DONT_FRAGMENT
ICMP echo_request checksum 0xd04c id 24662
00:10:11:809917: ip4-midchain
tx_sw_if_index 3 dpo-idx 4 : ipv4 via 0.0.0.0 ipip0: mtu:9000 next:5 flags:[fixup-ip4o4 ] 45000000000000004004e5a6c0a80a01c0a80a02
stacked-on entry:16:
[@2]: ipv4 via 192.168.10.2 host-ifinit: mtu:9000 next:4 flags:[] 02fe7293683f02fe45db61d20800 flow hash: 0x00000000
00000000: 45000068000000004004e53ec0a80a01c0a80a02450000548b2d40003f012727
00000020: c0a80502c0a803020800d04c605600021d311f6600000000bdf00e00
00:10:11:809919: esp4-encrypt-tun
esp: sa-index 0 spi 3867848325 (0xe68aae85) seq 13 sa-seq-hi 0 crypto aes-gcm-256 integrity none
00:10:11:809923: adj-midchain-tx
adj-midchain:[4]:ipv4 via 0.0.0.0 ipip0: mtu:9000 next:5 flags:[fixup-ip4o4 ] 45000000000000004004e5a6c0a80a01c0a80a02
stacked-on entry:16:
[@2]: ipv4 via 192.168.10.2 host-ifinit: mtu:9000 next:4 flags:[] 02fe7293683f02fe45db61d20800
00:10:11:809925: ip4-rewrite
tx_sw_if_index 1 dpo-idx 5 : ipv4 via 192.168.10.2 host-ifinit: mtu:9000 next:4 flags:[] 02fe7293683f02fe45db61d20800 flow hash: 0x00000000
00000000: 02fe7293683f02fe45db61d208004500008c000000003f32e5ecc0a80a01c0a8
00000020: 0a02e68aae850000000d0015fe5bd94666b31565f7358c46b0979172
00:10:11:809926: host-ifinit-output
host-ifinit flags 0x00180001
IP4: 02:fe:45:db:61:d2 -> 02:fe:72:93:68:3f
IPSEC_ESP: 192.168.10.1 -> 192.168.10.2
tos 0x00, ttl 63, length 140, checksum 0xe5ec dscp CS0 ecn NON_ECN
fragment id 0x0000
00:10:11:809927: host-ifinit-tx
af_packet: hw_if_index 1 tx-queue 0
tpacket3_hdr:
status 0x1 len 164 snaplen 164 mac 0 net 0
sec 0x0 nsec 0x0 vlan 0 vlan_tpid 0
vnet-hdr:
flags 0x00 gso_type 0x00 hdr_len 0
gso_size 0 csum_start 0 csum_offset 0
buffer 0x94d9d:
current data -36, length 154, buffer-pool 0, ref-count 1, trace handle 0x1
l2-hdr-offset 0 l3-hdr-offset 14
IP4: 02:fe:45:db:61:d2 -> 02:fe:72:93:68:3f
IPSEC_ESP: 192.168.10.1 -> 192.168.10.2
tos 0x00, ttl 63, length 140, checksum 0xe5ec dscp CS0 ecn NON_ECN
fragment id 0x0000
ping请求报文由命名空间clientns，通过veth类型接口veth_client，发送到vpp-initiator中的主机接口host-client（接口索引2）。VPP的处理节点如下所示，echo请求最后由host-ifinit-tx节点发送，此时为ESP封装的数据。请求报文通过veth接口ifinit发送到ifresp接口。
af-packet-input--->ethernet-input--->ip4-input--->ip4-lookup
|
|
adj-midchain-tx<---esp4-encrypt-tun<---ip4-midchain<---|
|
|
|--->ip4-rewrite--->host-ifinit-output--->host-ifinit-tx
ESP报文的封装情况如下。在ip4-midchain节点中，增加外层IP头部，如下：
45000000000000004004e5a6c0a80a01c0a80a02
-- --------
|--IPv4 长度20字节。 | --------
| |
192.168.10.1 |
|
192.168.10.2
另外，增加二层头部，如下：
02fe7293683f02fe45db61d20800
------------ ----
| ------------ |
| | |
host-ifresp接口MAC | 三层IP协议
|
host-ifinit接口MAC地址
在接下来的esp4-encrypt-tun节点，完成加密，在节点ip4-rewrite的trace中可看到ESP头部数据，如下：
00000020: 0a02e68aae850000000d0015fe5bd94666b31565f7358c46b0979172
-------- -----------------------------------
| -------- |
| | |
ESP SPI 序列号 加密数据
实例vpp-responder上的ping请求报文trace信息。
vpp# trace add af-packet-input 100
vpp# show trace
Packet 3
00:12:16:844067: af-packet-input
af_packet: hw_if_index 1 rx-queue 0 next-index 4
block 57:
address 0x7efd78834000 version 2 seq_num 58 pkt_num 0
tpacket3_hdr:
status 0x20000001 len 154 snaplen 154 mac 92 net 106
sec 0x661f311d nsec 0x3ad7a9a0 vlan 0 vlan_tpid 0
vnet-hdr:
flags 0x00 gso_type 0x00 hdr_len 0
gso_size 0 csum_start 0 csum_offset 0
00:12:16:844075: ethernet-input
IP4: 02:fe:45:db:61:d2 -> 02:fe:72:93:68:3f
00:12:16:844078: ip4-input
IPSEC_ESP: 192.168.10.1 -> 192.168.10.2
tos 0x00, ttl 63, length 140, checksum 0xe5ec dscp CS0 ecn NON_ECN
fragment id 0x0000
00:12:16:844081: ip4-lookup
fib 0 dpo-idx 7 flow hash: 0x00000000
IPSEC_ESP: 192.168.10.1 -> 192.168.10.2
tos 0x00, ttl 63, length 140, checksum 0xe5ec dscp CS0 ecn NON_ECN
fragment id 0x0000
00:12:16:844083: ip4-receive
IPSEC_ESP: 192.168.10.1 -> 192.168.10.2
tos 0x00, ttl 63, length 140, checksum 0xe5ec dscp CS0 ecn NON_ECN
fragment id 0x0000
00:12:16:844085: ipsec4-tun-input
IPSec: remote:192.168.10.1 spi:3867848325 (0xe68aae85) sa:1 tun:0 seq 13 sa -1035301868
00:12:16:844086: esp4-decrypt-tun
esp: crypto aes-gcm-256 integrity none pkt-seq 13 sa-seq 13 sa-seq-hi 0 pkt-seq-hi 0
00:12:16:844091: ip4-input-no-checksum
ICMP: 192.168.5.2 -> 192.168.3.2
tos 0x00, ttl 63, length 84, checksum 0x2727 dscp CS0 ecn NON_ECN
fragment id 0x8b2d, flags DONT_FRAGMENT
ICMP echo_request checksum 0xd04c id 24662
00:12:16:844092: ip4-lookup
fib 0 dpo-idx 6 flow hash: 0x00000000
ICMP: 192.168.5.2 -> 192.168.3.2
tos 0x00, ttl 63, length 84, checksum 0x2727 dscp CS0 ecn NON_ECN
fragment id 0x8b2d, flags DONT_FRAGMENT
ICMP echo_request checksum 0xd04c id 24662
00:12:16:844092: ip4-rewrite
tx_sw_if_index 2 dpo-idx 6 : ipv4 via 192.168.3.2 host-server: mtu:9000 next:6 flags:[] 9a3213ead1c502fe170ebb1f0800 flow hash: 0x00000000
00000000: 9a3213ead1c502fe170ebb1f0800450000548b2d40003e012827c0a80502c0a8
00000020: 03020800d04c605600021d311f6600000000bdf00e00000000001011
00:12:16:844094: host-server-output
host-server flags 0x00180001
IP4: 02:fe:17:0e:bb:1f -> 9a:32:13:ea:d1:c5
ICMP: 192.168.5.2 -> 192.168.3.2
tos 0x00, ttl 62, length 84, checksum 0x2827 dscp CS0 ecn NON_ECN
fragment id 0x8b2d, flags DONT_FRAGMENT
ICMP echo_request checksum 0xd04c id 24662
00:12:16:844095: host-server-tx
af_packet: hw_if_index 2 tx-queue 0
tpacket3_hdr:
status 0x1 len 108 snaplen 108 mac 0 net 0
sec 0x0 nsec 0x0 vlan 0 vlan_tpid 0
vnet-hdr:
flags 0x00 gso_type 0x00 hdr_len 0
gso_size 0 csum_start 0 csum_offset 0
buffer 0x94eea:
current data 36, length 98, buffer-pool 0, ref-count 1, trace handle 0x2
l2-hdr-offset 0 l3-hdr-offset 14
IP4: 02:fe:17:0e:bb:1f -> 9a:32:13:ea:d1:c5
ICMP: 192.168.5.2 -> 192.168.3.2
tos 0x00, ttl 62, length 84, checksum 0x2827 dscp CS0 ecn NON_ECN
fragment id 0x8b2d, flags DONT_FRAGMENT
ICMP echo_request checksum 0xd04c id 24662
ping请求报文通过veth类型接口ifresp，发送到vpp-responder中的主机接口host-ifresp（接口索引1）。VPP的处理节点如下所示，echo请求最后由host-server-tx节点发送，此时解密为普通的ping报文。请求报文通过veth接口server发送到命名空间serverns中的veth_server接口。
af-packet-input--->ethernet-input--->ip4-input--->ip4-lookup--->ip4-receive
|
|
--------esp4-decrypt-tun<---ipsec4-tun-input<---|
|
|
|--->ip4-input-no-checksum--->ip4-lookup--->ip4-rewrite--->host-server-output--->host-server-tx
ping回复报文的trace信息，vpp-responder先接收到回复报文，host-interface类型接口host-server（索引2）先接收到回复报文。
Packet 4
00:12:16:848038: af-packet-input
af_packet: hw_if_index 2 rx-queue 0 next-index 4
block 23:
address 0x7efd73a14000 version 2 seq_num 24 pkt_num 0
tpacket3_hdr:
status 0x20000001 len 98 snaplen 98 mac 92 net 106
sec 0x661f311d nsec 0x3b51a6ff vlan 0 vlan_tpid 0
vnet-hdr:
flags 0x00 gso_type 0x00 hdr_len 0
gso_size 0 csum_start 0 csum_offset 0
00:12:16:848042: ethernet-input
IP4: 9a:32:13:ea:d1:c5 -> 02:fe:17:0e:bb:1f
00:12:16:848044: ip4-input
ICMP: 192.168.3.2 -> 192.168.5.2
tos 0x00, ttl 64, length 84, checksum 0x5864 dscp CS0 ecn NON_ECN
fragment id 0x98f0
ICMP echo_reply checksum 0xd84c id 24662
00:12:16:848045: ip4-lookup
fib 0 dpo-idx 4 flow hash: 0x00000000
ICMP: 192.168.3.2 -> 192.168.5.2
tos 0x00, ttl 64, length 84, checksum 0x5864 dscp CS0 ecn NON_ECN
fragment id 0x98f0
ICMP echo_reply checksum 0xd84c id 24662
00:12:16:848046: ip4-midchain
tx_sw_if_index 3 dpo-idx 4 : ipv4 via 0.0.0.0 ipip0: mtu:9000 next:5 flags:[fixup-ip4o4 ] 45000000000000004004e5a6c0a80a02c0a80a01
stacked-on entry:16:
[@2]: ipv4 via 192.168.10.1 host-ifresp: mtu:9000 next:4 flags:[] 02fe45db61d202fe7293683f0800 flow hash: 0x00000000
00000000: 45000068000000004004e53ec0a80a02c0a80a014500005498f000003f015964
00000020: c0a80302c0a805020000d84c605600021d311f6600000000bdf00e00
00:12:16:848047: esp4-encrypt-tun
esp: sa-index 0 spi 1314930478 (0x4e603f2e) seq 12 sa-seq-hi 0 crypto aes-gcm-256 integrity none
00:12:16:848050: adj-midchain-tx
adj-midchain:[4]:ipv4 via 0.0.0.0 ipip0: mtu:9000 next:5 flags:[fixup-ip4o4 ] 45000000000000004004e5a6c0a80a02c0a80a01
stacked-on entry:16:
[@2]: ipv4 via 192.168.10.1 host-ifresp: mtu:9000 next:4 flags:[] 02fe45db61d202fe7293683f0800
00:12:16:848050: ip4-rewrite
tx_sw_if_index 1 dpo-idx 5 : ipv4 via 192.168.10.1 host-ifresp: mtu:9000 next:4 flags:[] 02fe45db61d202fe7293683f0800 flow hash: 0x00000000
00000000: 02fe45db61d202fe7293683f08004500008c000000003f32e5ecc0a80a02c0a8
00000020: 0a014e603f2e0000000cb98a3614bf8ed89ed439e169a4e237461279
00:12:16:848051: host-ifresp-output
host-ifresp flags 0x00180001
IP4: 02:fe:72:93:68:3f -> 02:fe:45:db:61:d2
IPSEC_ESP: 192.168.10.2 -> 192.168.10.1
tos 0x00, ttl 63, length 140, checksum 0xe5ec dscp CS0 ecn NON_ECN
fragment id 0x0000
00:12:16:848052: host-ifresp-tx
af_packet: hw_if_index 1 tx-queue 0
tpacket3_hdr:
status 0x1 len 164 snaplen 164 mac 0 net 0
sec 0x0 nsec 0x0 vlan 0 vlan_tpid 0
vnet-hdr:
flags 0x00 gso_type 0x00 hdr_len 0
gso_size 0 csum_start 0 csum_offset 0
buffer 0x94ec5:
current data -36, length 154, buffer-pool 0, ref-count 1, trace handle 0x3
l2-hdr-offset 0 l3-hdr-offset 14
IP4: 02:fe:72:93:68:3f -> 02:fe:45:db:61:d2
IPSEC_ESP: 192.168.10.2 -> 192.168.10.1
tos 0x00, ttl 63, length 140, checksum 0xe5ec dscp CS0 ecn NON_ECN
fragment id 0x0000
注意与ping请求报文不同，这里使用的为回复方向的ESP SPI（0x4e603f2e），不同于ping请求使用的（0xe68aae85）。
00000020: 0a014e603f2e0000000cb98a3614bf8ed89ed439e169a4e237461279
-------- -----------------------------------
| -------- |
| | |
ESP SPI 序列号 加密数据
vpp-initiator接收到ping回复报文的trace信息，报文由host-interface类型接口host-ifinit（索引1）进入VPP。最后由节点host-client-tx发送出去，命名空间clientns接收到ping回复报文。
Packet 3
00:10:11:825862: af-packet-input
af_packet: hw_if_index 1 rx-queue 0 next-index 4
block 54:
address 0x7f124ac44000 version 2 seq_num 55 pkt_num 0
tpacket3_hdr:
status 0x20000001 len 154 snaplen 154 mac 92 net 106
sec 0x661f311d nsec 0x3b8dc4bf vlan 0 vlan_tpid 0
vnet-hdr:
flags 0x00 gso_type 0x00 hdr_len 0
gso_size 0 csum_start 0 csum_offset 0
00:10:11:825867: ethernet-input
IP4: 02:fe:72:93:68:3f -> 02:fe:45:db:61:d2
00:10:11:825870: ip4-input
IPSEC_ESP: 192.168.10.2 -> 192.168.10.1
tos 0x00, ttl 63, length 140, checksum 0xe5ec dscp CS0 ecn NON_ECN
fragment id 0x0000
00:10:11:825871: ip4-lookup
fib 0 dpo-idx 7 flow hash: 0x00000000
IPSEC_ESP: 192.168.10.2 -> 192.168.10.1
tos 0x00, ttl 63, length 140, checksum 0xe5ec dscp CS0 ecn NON_ECN
fragment id 0x0000
00:10:11:825872: ip4-receive
IPSEC_ESP: 192.168.10.2 -> 192.168.10.1
tos 0x00, ttl 63, length 140, checksum 0xe5ec dscp CS0 ecn NON_ECN
fragment id 0x0000
00:10:11:825874: ipsec4-tun-input
IPSec: remote:192.168.10.2 spi:1314930478 (0x4e603f2e) sa:1 tun:0 seq 12 sa -1804485612
00:10:11:825875: esp4-decrypt-tun
esp: crypto aes-gcm-256 integrity none pkt-seq 12 sa-seq 12 sa-seq-hi 0 pkt-seq-hi 0
00:10:11:825878: ip4-input-no-checksum
ICMP: 192.168.3.2 -> 192.168.5.2
tos 0x00, ttl 63, length 84, checksum 0x5964 dscp CS0 ecn NON_ECN
fragment id 0x98f0
ICMP echo_reply checksum 0xd84c id 24662
00:10:11:825879: ip4-lookup
fib 0 dpo-idx 6 flow hash: 0x00000000
ICMP: 192.168.3.2 -> 192.168.5.2
tos 0x00, ttl 63, length 84, checksum 0x5964 dscp CS0 ecn NON_ECN
fragment id 0x98f0
ICMP echo_reply checksum 0xd84c id 24662
00:10:11:825880: ip4-rewrite
tx_sw_if_index 2 dpo-idx 6 : ipv4 via 192.168.5.2 host-client: mtu:9000 next:6 flags:[] 9685ba4b2a9002fe017471410800 flow hash: 0x00000000
00000000: 9685ba4b2a9002fe0174714108004500005498f000003e015a64c0a80302c0a8
00000020: 05020000d84c605600021d311f6600000000bdf00e00000000001011
00:10:11:825882: host-client-output
host-client flags 0x00180001
IP4: 02:fe:01:74:71:41 -> 96:85:ba:4b:2a:90
ICMP: 192.168.3.2 -> 192.168.5.2
tos 0x00, ttl 62, length 84, checksum 0x5a64 dscp CS0 ecn NON_ECN
fragment id 0x98f0
ICMP echo_reply checksum 0xd84c id 24662
00:10:11:825883: host-client-tx
af_packet: hw_if_index 2 tx-queue 0
tpacket3_hdr:
status 0x1 len 108 snaplen 108 mac 0 net 0
sec 0x0 nsec 0x0 vlan 0 vlan_tpid 0
vnet-hdr:
flags 0x00 gso_type 0x00 hdr_len 0
gso_size 0 csum_start 0 csum_offset 0
buffer 0x94d78:
current data 36, length 98, buffer-pool 0, ref-count 1, trace handle 0x2
l2-hdr-offset 0 l3-hdr-offset 14
IP4: 02:fe:01:74:71:41 -> 96:85:ba:4b:2a:90
ICMP: 192.168.3.2 -> 192.168.5.2
tos 0x00, ttl 62, length 84, checksum 0x5a64 dscp CS0 ecn NON_ECN
fragment id 0x98f0
ICMP echo_reply checksum 0xd84c id 24662
vpp-initiator报文处理流程
vpp-initiator实例在接收到192.168.5.2 ping 192.168.3.2的请求报文后，在ip4-lookup节点查询路由，如下路由表项15，由CLI命令行配置生成。下一跳网关为192.168.10.2，出接口为ipip0，DPO类型为DPO_ADJACENCY_MIDCHAIN，rewrite为IPv4头部数据：45000000000000004004e5a6c0a80a01c0a80a02。
下一个处理节点为ip4-midchain，即dpoi_next_node为6（[0] [@6]: ipv4），通过命令show node ip4-lookup可查看6对应于ip4-midchain节点。
vpp# show fib entry 15
15@192.168.3.0/24 fib:0 index:15 locks:2
CLI refs:1 src-flags:added,contributing,active,
path-list:[20] locks:2 flags:shared, uPRF-list:23 len:1 itfs:[3, ]
path:[24] pl-index:20 ip4 weight=1 pref=0 attached-nexthop: oper-flags:resolved,
192.168.10.2 ipip0 (p2p)
[@0]: ipv4 via 0.0.0.0 ipip0: mtu:9000 next:5 flags:[fixup-ip4o4 ] 45000000000000004004e5a6c0a80a01c0a80a02
stacked-on entry:16:
[@2]: ipv4 via 192.168.10.2 host-ifinit: mtu:9000 next:4 flags:[] 02fed0bc516402fe0141578c0800
forwarding: unicast-ip4-chain
[@0]: dpo-load-balance: [proto:ip4 index:17 buckets:1 uRPF:23 to:[9859:828156]]
[0] [@6]: ipv4 via 0.0.0.0 ipip0: mtu:9000 next:5 flags:[fixup-ip4o4 ] 45000000000000004004e5a6c0a80a01c0a80a02
stacked-on entry:16:
[@2]: ipv4 via 192.168.10.2 host-ifinit: mtu:9000 next:4 flags:[] 02fed0bc516402fe0141578c0800
FIB表项15依托于FIB表项16，DPO类型为DPO_ADJACENCY，rewrite为二层头部数据：02fed0bc516402fe0141578c0800。
vpp# show fib entry 16
16@192.168.10.2/32 fib:0 index:16 locks:4
adjacency refs:1 entry-flags:attached, src-flags:added,contributing,active, cover:7
path-list:[22] locks:2 uPRF-list:22 len:1 itfs:[1, ]
path:[26] pl-index:22 ip4 weight=1 pref=0 attached-nexthop: oper-flags:resolved,
192.168.10.2 host-ifinit
[@0]: ipv4 via 192.168.10.2 host-ifinit: mtu:9000 next:4 flags:[] 02fed0bc516402fe0141578c0800
Extensions:
path:26 adj-flags:[refines-cover]
recursive-resolution refs:1 src-flags:added, cover:-1
forwarding: unicast-ip4-chain
[@0]: dpo-load-balance: [proto:ip4 index:18 buckets:1 uRPF:22 to:[905:91028]]
[0] [@5]: ipv4 via 192.168.10.2 host-ifinit: mtu:9000 next:4 flags:[] 02fed0bc516402fe0141578c0800
Delegates:
track: sibling:34
Children:{adj:4}
Children:{fib-entry-track:2}
如上FIB表现16，关联的邻居表项索引为4（Children:{adj:4}）。
vpp# show adj 4
[@4] ipv4 via 0.0.0.0 ipip0: mtu:9000 next:5 flags:[fixup-ip4o4 ] 45000000000000004004e5a6c0a80a01c0a80a02
stacked-on entry:16:
[@2]: ipv4 via 192.168.10.2 host-ifinit: mtu:9000 next:4 flags:[] 02fed0bc516402fe0141578c0800
flags:midchain-ip-stack midchain-ip4o4-hdr-fixup
counts:[0:0]
locks:3
delegates:
{MIDCHAIN:[fib-entry:16]}
{ipsec-tun-protect:
ipip0 flags:[none]
output-sa:
[0] sa 2147483648 (0x80000000) spi 351754080 (0x14f75760) protocol:esp flags:[esn anti-replay aead ctr ]
input-sa:
[1] sa 3221225472 (0xc0000000) spi 1831487862 (0x6d2a4976) protocol:esp flags:[esn anti-replay inbound aead ctr ]}
children:
{path:24}
在节点ip4-midchain，为报文封装上IPSec的外部的IPv4头数据：45000000000000004004e5a6c0a80a01c0a80a02。并且更新其中的IP总长度，IP头部校验和等字段。下一个处理节点的索引为5（参见以上adj索引4：[@4] ipv4 via 0.0.0.0 ipip0: mtu:9000 next:5），可通过show node ip4-midchain查看索引5的节点为esp4-encrypt-tun。
在节点esp4-encrypt-tun，根据邻居adj 4，取得output-sa，加密数据，添加ESP头部。下一个节点为adj-midchain-tx。
VLIB_REGISTER_NODE (esp4_encrypt_tun_node) = {
.name = "esp4-encrypt-tun",
.vector_size = sizeof (u32),
.format_trace = format_esp_encrypt_trace,
.n_next_nodes = ESP_ENCRYPT_N_NEXT,
.next_nodes = {
[ESP_ENCRYPT_NEXT_INTERFACE_OUTPUT] = "adj-midchain-tx",
},
};
在节点adj-midchain-tx中，找到下一个DPO的dpoi_next_node，这里为2（[@2]），参考show node adj-midchain-tx，下一个节点为ip4-rewrite。
vpp# show adj 4
[@4] ipv4 via 0.0.0.0 ipip0: mtu:9000 next:5 flags:[fixup-ip4o4 ] 45000000000000004004e5a6c0a80a01c0a80a02
stacked-on entry:16:
[@2]: ipv4 via 192.168.10.2 host-ifinit: mtu:9000 next:4 flags:[] 02fed0bc516402fe0141578c0800
vpp# show node adj-midchain-tx
node adj-midchain-tx, type internal, state active, index 402
next nodes:
next-index node-index Node Vectors
0 690 error-drop 0
1 622 ip4-drop 0
2 630 ip4-rewrite 0
重要的在在adj-midchain-tx中更换邻居索引，更换成midchain依托的DPO。dpo0->dpoi_index的值为5。
always_inline uword
adj_midchain_tx_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
vlib_frame_t * frame, int interface_count)
{
adj_index0 = vnet_buffer(b[0])->ip.adj_index[VLIB_TX];
adj0 = adj_get(adj_index0);
dpo0 = &adj0->sub_type.midchain.next_dpo;
next[0] = dpo0->dpoi_next_node;
vnet_buffer(b[0])->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
邻居表项5的内容如下。注意下一个节点的索引为4（next:4）。
vpp# show adj 5
[@5] ipv4 via 192.168.10.2 host-ifinit: mtu:9000 next:4 flags:[] 02fed0bc516402fe0141578c0800
flags:None
counts:[0:0]
locks:4
delegates:
children:
{path:26}
在节点ip4-rewrite中，为ESP报文增加二层头部数据：02fed0bc516402fe0141578c0800，交由节点4(host-ifinit-output)发送出去。最终通过veth接口发送到ifresp，vpp-responder实例的host-ifresp接口将接收到报文。
vpp# show node ip4-rewrite
node ip4-rewrite, type internal, state active, index 630
next nodes:
next-index node-index Node Vectors
0 622 ip4-drop 0
1 638 ip4-icmp-error 0
2 559 ip4-frag 0
3 401 tunnel-output 0
4 708 host-ifinit-output 24
vpp-initiator报文处理流程
vpp-initiator实例在接收到192.168.10.1到192.168.10.2的ESP报文后，在ip4-lookup节点查询路由，如下路由表项15，根据接口host-ifresp地址生成的路由项。目的地址是本机。
vpp# show fib entry 10
10@192.168.10.2/32 fib:0 index:10 locks:2
interface refs:1 entry-flags:connected,local, src-flags:added,contributing,active, cover:7
path-list:[17] locks:2 flags:local, uPRF-list:15 len:0 itfs:[]
path:[19] pl-index:17 ip4 weight=1 pref=0 receive: oper-flags:resolved, cfg-flags:local,
[@0]: dpo-receive: 192.168.10.2 on host-ifresp
forwarding: unicast-ip4-chain
[@0]: dpo-load-balance: [proto:ip4 index:12 buckets:1 uRPF:15 to:[1555:214028]]
[0] [@12]: dpo-receive: 192.168.10.2 on host-ifresp
Delegates:
Children:
下一个处理节点为ip4-receive，即dpoi_next_node为12（[0] [@12]: dpo-receive），通过命令show node ip4-lookup可查看12对应于ip4-receive节点。
vpp# show node ip4-lookup
node ip4-lookup, type internal, state active, index 635
next nodes:
next-index node-index Node Vectors
4 536 ip4-glean 1
5 630 ip4-rewrite 3036
6 626 ip4-midchain 1465
12 632 ip4-receive 1572
在ip4-receive节点，由于ipsec_tun_register_nodes函数注册了ESP报文处理节点ipsec4_tun_input_node，将报文转到ipsec4_tun_input_node处理。
void ipsec_tun_register_nodes (ip_address_family_t af)
{
if (0 == ipsec_tun_node_regs[af]++)
{
if (AF_IP4 == af)
ip4_register_protocol (IP_PROTOCOL_IPSEC_ESP,
ipsec4_tun_input_node.index);
在ipsec4_tun_input节点，根据源地址:192.168.10.1和SPI:4102132475 (0xf48192fb)，找到SA，索引1。到下一个节点esp4-decrypt-tun。
vpp# show ipsec sa
[0] sa 2147483648 (0x80000000) spi 2422636174 (0x90667e8e) protocol:esp flags:[esn anti-replay aead ctr ]
[1] sa 3221225472 (0xc0000000) spi 4102132475 (0xf48192fb) protocol:esp flags:[esn anti-replay inbound aead ctr ]
在节点esp4-decrypt-tun解密数据包，去掉外层IPv4头部，转到ip4-input-no-checksum节点处理。此时报文为ICMP: 192.168.5.2 -> 192.168.3.2，转到ip4-lookup节点处理，查询路由表项17，找到DPO：[@5]: ipv4 via 192.168.3.2 host-server: mtu:9000 next:6。所以下一节点索引为5（[@5]），即ip4-rewrite。
vpp# show fib entry 17
17@192.168.3.2/32 fib:0 index:17 locks:2
adjacency refs:1 entry-flags:attached, src-flags:added,contributing,active, cover:11
path-list:[23] locks:2 uPRF-list:20 len:1 itfs:[2, ]
path:[27] pl-index:23 ip4 weight=1 pref=0 attached-nexthop: oper-flags:resolved,
192.168.3.2 host-server
[@0]: ipv4 via 192.168.3.2 host-server: mtu:9000 next:6 flags:[] 9a3213ead1c502fe39d1fe6d0800
Extensions:
path:27 adj-flags:[refines-cover]
forwarding: unicast-ip4-chain
[@0]: dpo-load-balance: [proto:ip4 index:19 buckets:1 uRPF:20 to:[3446:289464]]
[0] [@5]: ipv4 via 192.168.3.2 host-server: mtu:9000 next:6 flags:[] 9a3213ead1c502fe39d1fe6d0800
Delegates:
Children:
在节点ip4-rewrite中，重新二层头部数据：9a3213ead1c502fe39d1fe6d0800，转到下一个节点6（host-server: mtu:9000 next:6），即host-server-output。
vpp# show node ip4-rewrite
node ip4-rewrite, type internal, state active, index 630
next nodes:
next-index node-index Node Vectors
4 708 host-ifresp-output 10486
6 710 host-server-output 10107
报文通过veth接口对，发送到命名空间serverns中的veth_server接口。ping回复报文流程与以上类似。
