OVS-DPDK and NUMA awareness do not seem to be working in Red Hat OpenStack Platform

Issue

OVS-DPDK and NUMA awareness in Red Hat OpenStack Platform do not seem to be working with OVS 2.6.1

This can be reproduced this with latest OVS 2.6.1-10 and DPDK 16.11-4

Verification with a script that is launching qemu-kvm processes and creates the vhostuser ports.

[root@compute1 ~]# cat ./startvm.sh
#!/bin/bash -x

export VM_NAME0=vhost-vm0
export DPDKPORT0=dpdkvhostuser0
export VM_NAME1=vhost-vm1
export DPDKPORT1=dpdkvhostuser1
export GUEST_MEM=3072M
export QCOW2_IMAGE=/root/CentOS7_x86_64.qcow2
export VHOST_SOCK_DIR=/var/run/openvswitch
export BRIDGE_NAME=br-test

echo "Creating ports on $BDIGE_NAME"
ovs-vsctl add-port $BRIDGE_NAME dpdkvhostuser0 -- set Interface dpdkvhostuser0 type=dpdkvhostuser ofport_request=3
ovs-vsctl add-port $BRIDGE_NAME dpdkvhostuser1 -- set Interface dpdkvhostuser1 type=dpdkvhostuser ofport_request=4
ovs-ofctl show br-test

taskset -c 3,31 /usr/libexec/qemu-kvm -name $VM_NAME0 -cpu host -enable-kvm \
  -m $GUEST_MEM -drive file=$QCOW2_IMAGE  -snapshot \
  -numa node,memdev=mem -mem-prealloc -smp sockets=1,cores=2 \
  -object memory-backend-file,id=mem,size=$GUEST_MEM,mem-path=/dev/hugepages,share=on \
  -chardev socket,id=char0,path=$VHOST_SOCK_DIR/$DPDKPORT0 \
  -netdev type=vhost-user,id=mynet1,chardev=char0,vhostforce \
  -device virtio-net-pci,mac=fa:16:3e:aa:bb:01,netdev=mynet1,mrg_rxbuf=off -daemonize -vnc 0.0.0.0:0

taskset -c 17,45 /usr/libexec/qemu-kvm -name $VM_NAME1 -cpu host -enable-kvm \
  -m $GUEST_MEM -drive file=$QCOW2_IMAGE  -snapshot \
  -numa node,memdev=mem -mem-prealloc -smp sockets=1,cores=2 \
  -object memory-backend-file,id=mem,size=$GUEST_MEM,mem-path=/dev/hugepages,share=on \
  -chardev socket,id=char0,path=$VHOST_SOCK_DIR/$DPDKPORT1 \
  -netdev type=vhost-user,id=mynet2,chardev=char0,vhostforce \
  -device virtio-net-pci,mac=fa:16:3e:aa:bb:02,netdev=mynet2,mrg_rxbuf=off -daemonize -vnc 0.0.0.0:1

sleep 15
 pgrep qemu-kvm | while read pid;do echo "threads for pid $pid";echo "==="; ps aux | awk "\$2 == $pid"; ps -T -p $pid ; done
 pgrep qemu-kvm | while read pid;do echo "taskset for pid $pid";echo "==="; ps aux | awk "\$2 == $pid"; ps -T -p $pid | awk '$2 != "SPID" {print $2}' | xargs -I {} taskset -cp {};done
ovs-appctl dpif-netdev/pmd-rxq-show

echo "tuna -t ovs-vswitchd -CP | grep pmd output"
tuna -t ovs-vswitchd -CP | grep pmd

echo "Type RETURN when you are done ..."
read RETURN

echo "Cleaning up"
kill `ps aux | grep qemu-kvm | grep $VM_NAME0 | awk '{print $2}'`
kill `ps aux | grep qemu-kvm | grep $VM_NAME1 | awk '{print $2}'`
ovs-vsctl del-port  $BRIDGE_NAME dpdkvhostuser0
ovs-vsctl del-port  $BRIDGE_NAME dpdkvhostuser1

The NUMA architecture of the node looks like this:

[root@compute1 qemu]# numactl -H
available: 2 nodes (0-1)
node 0 cpus: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 28 29 30 31 32 33 34 35 36 37 38 39 40 41
node 0 size: 130799 MB
node 0 free: 29048 MB
node 1 cpus: 14 15 16 17 18 19 20 21 22 23 24 25 26 27 42 43 44 45 46 47 48 49 50 51 52 53 54 55
node 1 size: 131072 MB
node 1 free: 29215 MB
node distances:
node   0   1
  0:  10  21
  1:  21  10

The script yields the following

[root@compute1 ~]# ./startvm.sh
+ export VM_NAME0=vhost-vm0
+ VM_NAME0=vhost-vm0
+ export DPDKPORT0=dpdkvhostuser0
+ DPDKPORT0=dpdkvhostuser0
+ export VM_NAME1=vhost-vm1
+ VM_NAME1=vhost-vm1
+ export DPDKPORT1=dpdkvhostuser1
+ DPDKPORT1=dpdkvhostuser1
+ export GUEST_MEM=3072M
+ GUEST_MEM=3072M
+ export QCOW2_IMAGE=/root/CentOS7_x86_64.qcow2
+ QCOW2_IMAGE=/root/CentOS7_x86_64.qcow2
+ export VHOST_SOCK_DIR=/var/run/openvswitch
+ VHOST_SOCK_DIR=/var/run/openvswitch
+ export BRIDGE_NAME=br-test
+ BRIDGE_NAME=br-test
+ echo 'Creating ports on '
Creating ports on
+ ovs-vsctl add-port br-test dpdkvhostuser0 -- set Interface dpdkvhostuser0 type=dpdkvhostuser ofport_request=3
+ ovs-vsctl add-port br-test dpdkvhostuser1 -- set Interface dpdkvhostuser1 type=dpdkvhostuser ofport_request=4
+ ovs-ofctl show br-test
OFPT_FEATURES_REPLY (xid=0x2): dpid:00000025b581006f
n_tables:254, n_buffers:256
capabilities: FLOW_STATS TABLE_STATS PORT_STATS QUEUE_STATS ARP_MATCH_IP
actions: output enqueue set_vlan_vid set_vlan_pcp strip_vlan mod_dl_src mod_dl_dst mod_nw_src mod_nw_dst mod_nw_tos mod_tp_src mod_tp_dst
 1(phy-br-test): addr:ff:ee:dd:cc:bb:aa
     config:     0
     state:      0
     speed: 0 Mbps now, 0 Mbps max
 2(dpdk0): addr:aa:bb:cc:dd:ee:ff
     config:     0
     state:      0
     current:    1GB-HD 1GB-FD
     speed: 1000 Mbps now, 0 Mbps max
 3(dpdkvhostuser0): addr:00:00:00:00:00:00
     config:     0
     state:      LINK_DOWN
     speed: 0 Mbps now, 0 Mbps max
 4(dpdkvhostuser1): addr:00:00:00:00:00:00
     config:     0
     state:      LINK_DOWN
     speed: 0 Mbps now, 0 Mbps max
 LOCAL(br-test): addr:aa:bb:cc:dd:ee:ff
     config:     0
     state:      0
     current:    10MB-FD COPPER
     speed: 10 Mbps now, 0 Mbps max
OFPT_GET_CONFIG_REPLY (xid=0x4): frags=normal miss_send_len=0
+ taskset -c 3,31 /usr/libexec/qemu-kvm -name vhost-vm0 -cpu host -enable-kvm -m 3072M -drive file=/root/CentOS7_x86_64.qcow2 -snapshot -numa node,memdev=mem -mem-prealloc -smp sockets=1,cores=2 -object memory-backend-file,id=mem,size=3072M,mem-path=/dev/hugepages,share=on -chardev socket,id=char0,path=/var/run/openvswitch/dpdkvhostuser0 -netdev type=vhost-user,id=mynet1,chardev=char0,vhostforce -device virtio-net-pci,mac=fa:16:3e:aa:bb:01,netdev=mynet1,mrg_rxbuf=off -daemonize -vnc 0.0.0.0:0
+ taskset -c 17,45 /usr/libexec/qemu-kvm -name vhost-vm1 -cpu host -enable-kvm -m 3072M -drive file=/root/CentOS7_x86_64.qcow2 -snapshot -numa node,memdev=mem -mem-prealloc -smp sockets=1,cores=2 -object memory-backend-file,id=mem,size=3072M,mem-path=/dev/hugepages,share=on -chardev socket,id=char0,path=/var/run/openvswitch/dpdkvhostuser1 -netdev type=vhost-user,id=mynet2,chardev=char0,vhostforce -device virtio-net-pci,mac=fa:16:3e:aa:bb:02,netdev=mynet2,mrg_rxbuf=off -daemonize -vnc 0.0.0.0:1
+ sleep 15
+ pgrep qemu-kvm
+ read pid
+ echo 'threads for pid 11546'
threads for pid 11546
+ echo ===
===
+ ps aux
+ awk '$2 == 11546'
root       11546 95.3  0.0 4070812 23980 ?       Sl   18:03   0:16 /usr/libexec/qemu-kvm -name vhost-vm0 -cpu host -enable-kvm -m 3072M -drive file=/root/CentOS7_x86_64.qcow2 -snapshot -numa node,memdev=mem -mem-prealloc -smp sockets=1,cores=2 -object memory-backend-file,id=mem,size=3072M,mem-path=/dev/hugepages,share=on -chardev socket,id=char0,path=/var/run/openvswitch/dpdkvhostuser0 -netdev type=vhost-user,id=mynet1,chardev=char0,vhostforce -device virtio-net-pci,mac=fa:16:3e:aa:bb:01,netdev=mynet1,mrg_rxbuf=off -daemonize -vnc 0.0.0.0:0
+ ps -T -p 11546
    PID    SPID TTY          TIME CMD
  11546   11546 ?        00:00:02 qemu-kvm
  11546   11547 ?        00:00:00 qemu-kvm
  11546   11552 ?        00:00:00 qemu-kvm
  11546   11553 ?        00:00:10 qemu-kvm
  11546   11554 ?        00:00:03 qemu-kvm
  11546   11556 ?        00:00:00 qemu-kvm
+ read pid
+ echo 'threads for pid 11560'
threads for pid 11560
+ echo ===
===
+ ps aux
+ awk '$2 == 11560'
root       11560 94.2  0.0 4059552 27484 ?       Sl   18:03   0:15 /usr/libexec/qemu-kvm -name vhost-vm1 -cpu host -enable-kvm -m 3072M -drive file=/root/CentOS7_x86_64.qcow2 -snapshot -numa node,memdev=mem -mem-prealloc -smp sockets=1,cores=2 -object memory-backend-file,id=mem,size=3072M,mem-path=/dev/hugepages,share=on -chardev socket,id=char0,path=/var/run/openvswitch/dpdkvhostuser1 -netdev type=vhost-user,id=mynet2,chardev=char0,vhostforce -device virtio-net-pci,mac=fa:16:3e:aa:bb:02,netdev=mynet2,mrg_rxbuf=off -daemonize -vnc 0.0.0.0:1
+ ps -T -p 11560
    PID    SPID TTY          TIME CMD
  11560   11560 ?        00:00:02 qemu-kvm
  11560   11561 ?        00:00:00 qemu-kvm
  11560   11563 ?        00:00:00 qemu-kvm
  11560   11564 ?        00:00:08 qemu-kvm
  11560   11565 ?        00:00:03 qemu-kvm
  11560   11567 ?        00:00:00 qemu-kvm
+ read pid
+ pgrep qemu-kvm
+ read pid
+ echo 'taskset for pid 11546'
taskset for pid 11546
+ echo ===
===
+ ps aux
+ awk '$2 == 11546'
root       11546 95.6  0.0 4070812 23980 ?       Sl   18:03   0:16 /usr/libexec/qemu-kvm -name vhost-vm0 -cpu host -enable-kvm -m 3072M -drive file=/root/CentOS7_x86_64.qcow2 -snapshot -numa node,memdev=mem -mem-prealloc -smp sockets=1,cores=2 -object memory-backend-file,id=mem,size=3072M,mem-path=/dev/hugepages,share=on -chardev socket,id=char0,path=/var/run/openvswitch/dpdkvhostuser0 -netdev type=vhost-user,id=mynet1,chardev=char0,vhostforce -device virtio-net-pci,mac=fa:16:3e:aa:bb:01,netdev=mynet1,mrg_rxbuf=off -daemonize -vnc 0.0.0.0:0
+ ps -T -p 11546
+ awk '$2 != "SPID" {print $2}'
+ xargs -I '{}' taskset -cp '{}'
pid 11546's current affinity list: 3,31
pid 11547's current affinity list: 3,31
pid 11552's current affinity list: 3,31
pid 11553's current affinity list: 3,31
pid 11554's current affinity list: 3,31
pid 11556's current affinity list: 3,31
+ read pid
+ echo 'taskset for pid 11560'
taskset for pid 11560
+ echo ===
===
+ ps aux
+ awk '$2 == 11560'
root       11560 94.2  0.0 4059552 27484 ?       Sl   18:03   0:15 /usr/libexec/qemu-kvm -name vhost-vm1 -cpu host -enable-kvm -m 3072M -drive file=/root/CentOS7_x86_64.qcow2 -snapshot -numa node,memdev=mem -mem-prealloc -smp sockets=1,cores=2 -object memory-backend-file,id=mem,size=3072M,mem-path=/dev/hugepages,share=on -chardev socket,id=char0,path=/var/run/openvswitch/dpdkvhostuser1 -netdev type=vhost-user,id=mynet2,chardev=char0,vhostforce -device virtio-net-pci,mac=fa:16:3e:aa:bb:02,netdev=mynet2,mrg_rxbuf=off -daemonize -vnc 0.0.0.0:1
+ ps -T -p 11560
+ awk '$2 != "SPID" {print $2}'
+ xargs -I '{}' taskset -cp '{}'
pid 11560's current affinity list: 17,45
pid 11561's current affinity list: 17,45
pid 11563's current affinity list: 17,45
pid 11564's current affinity list: 17,45
pid 11565's current affinity list: 17,45
pid 11567's current affinity list: 17,45
+ read pid
+ ovs-appctl dpif-netdev/pmd-rxq-show
pmd thread numa_id 0 core_id 1:
        isolated : false
        port: dpdk1     queue-id: 0
        port: dpdkvhostuser0    queue-id: 0
pmd thread numa_id 0 core_id 29:
        isolated : false
        port: dpdk0     queue-id: 0
        port: dpdkvhostuser1    queue-id: 0
+ echo 'tuna -t ovs-vswitchd -CP | grep pmd output'
tuna -t ovs-vswitchd -CP | grep pmd output
+ tuna -t ovs-vswitchd -CP
+ grep pmd
  1665   OTHER     0        1         0        50381           pmd16
  1666   OTHER     0       29         0         2327           pmd17
+ echo 'Type RETURN when you are done ...'
Type RETURN when you are done ...
+ read RETURN

+ echo 'Cleaning up'
Cleaning up
++ ps aux
++ grep vhost-vm0
++ grep qemu-kvm
++ awk '{print $2}'
+ kill 11546
++ ps aux
++ grep vhost-vm1
++ grep qemu-kvm
++ awk '{print $2}'
+ kill 11560
+ ovs-vsctl del-port br-test dpdkvhostuser0
+ ovs-vsctl del-port br-test dpdkvhostuser1

Meaning that NUMA awareness is not working.