Chapter 5. Configuring SR-IOV and DPDK interfaces on the same compute node

Add additional resources.

  resources:
    userdata:
      type: OS::Heat::MultipartMime
      properties:
        parts:
        - config: {get_resource: set_dpdk_params}
        - config: {get_resource: install_tuned}
        - config: {get_resource: compute_kernel_args}

Set the DPDK parameters.

  set_dpdk_params:
    type: OS::Heat::SoftwareConfig
    properties:
      config:
        str_replace:
          template: |
            #!/bin/bash
            set -x
            get_mask()
            {
              local list=$1
              local mask=0
              declare -a bm
              max_idx=0
              for core in $(echo $list | sed 's/,/ /g')
              do
                  index=$(($core/32))
                  bm[$index]=0
                  if [ $max_idx -lt $index ]; then
                     max_idx=$(($index))
                  fi
              done
              for ((i=$max_idx;i>=0;i--));
              do
                  bm[$i]=0
              done
              for core in $(echo $list | sed 's/,/ /g')
              do
                  index=$(($core/32))
                  temp=$((1<<$(($core % 32))))
                  bm[$index]=$((${bm[$index]} | $temp))
              done

              printf -v mask "%x" "${bm[$max_idx]}"
              for ((i=$max_idx-1;i>=0;i--));
              do
                  printf -v hex "%08x" "${bm[$i]}"
                  mask+=$hex
              done
              printf "%s" "$mask"
            }

            FORMAT=$COMPUTE_HOSTNAME_FORMAT
            if [[ -z $FORMAT ]] ; then
              FORMAT="compute" ;
            else
              # Assumption: only %index% and %stackname% are the variables in Host name format
              FORMAT=$(echo $FORMAT | sed  's/\%index\%//g' | sed 's/\%stackname\%//g') ;
            fi
            if [[ $(hostname) == *$FORMAT* ]] ; then
              # 42477 is the kolla hugetlbfs gid value.
              getent group hugetlbfs >/dev/null || \
                groupadd hugetlbfs -g 42477 && groupmod -g 42477 hugetlbfs

              pmd_cpu_mask=$( get_mask $PMD_CORES )
              host_cpu_mask=$( get_mask $LCORE_LIST )
              socket_mem=$(echo $SOCKET_MEMORY | sed s/\'//g )
              ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-init=true
              ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-socket-mem=$socket_mem
              ovs-vsctl --no-wait set Open_vSwitch . other_config:pmd-cpu-mask=$pmd_cpu_mask
              ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-lcore-mask=$host_cpu_mask
            fi
          params:
            $COMPUTE_HOSTNAME_FORMAT: {get_param: ComputeHostnameFormat}
            $LCORE_LIST: {get_param: HostCpusList}
            $PMD_CORES: {get_param: NeutronDpdkCoreList}
            $SOCKET_MEMORY: {get_param: NeutronDpdkSocketMemory}

Set the tuned configuration to provide CPU affinity.

  install_tuned:
    type: OS::Heat::SoftwareConfig
    properties:
      config:
        str_replace:
          template: |
            #!/bin/bash
            FORMAT=$COMPUTE_HOSTNAME_FORMAT
            if [[ -z $FORMAT ]] ; then
              FORMAT="compute" ;
            else
              # Assumption: only %index% and %stackname% are the variables in Host name format
              FORMAT=$(echo $FORMAT | sed  's/\%index\%//g' | sed 's/\%stackname\%//g') ;
            fi
            if [[ $(hostname) == *$FORMAT* ]] ; then
              # Install the tuned package
              yum install -y tuned-profiles-cpu-partitioning

              tuned_conf_path="/etc/tuned/cpu-partitioning-variables.conf"
              if [ -n "$TUNED_CORES" ]; then
                grep -q "^isolated_cores" $tuned_conf_path
                if [ "$?" -eq 0 ]; then
                  sed -i 's/^isolated_cores=.*/isolated_cores=$TUNED_CORES/' $tuned_conf_path
                else
                  echo "isolated_cores=$TUNED_CORES" >> $tuned_conf_path
                fi
                tuned-adm profile cpu-partitioning
              fi
            fi
          params:
            $COMPUTE_HOSTNAME_FORMAT: {get_param: ComputeHostnameFormat}
            $TUNED_CORES: {get_param: HostIsolatedCoreList}

Set the kernel arguments.

  compute_kernel_args:
    type: OS::Heat::SoftwareConfig
    properties:
      config:
        str_replace:
          template: |
            #!/bin/bash
            FORMAT=$COMPUTE_HOSTNAME_FORMAT
            if [[ -z $FORMAT ]] ; then
              FORMAT="compute" ;
            else
              # Assumption: only %index% and %stackname% are the variables in Host name format
              FORMAT=$(echo $FORMAT | sed  's/\%index\%//g' | sed 's/\%stackname\%//g') ;
            fi
            if [[ $(hostname) == *$FORMAT* ]] ; then
              sed 's/^\(GRUB_CMDLINE_LINUX=".*\)"/\1 $KERNEL_ARGS isolcpus=$TUNED_CORES"/g' -i /etc/default/grub ;
              grub2-mkconfig -o /etc/grub2.cfg
              reboot
            fi
          params:
            $KERNEL_ARGS: {get_param: ComputeKernelArgs}
            $COMPUTE_HOSTNAME_FORMAT: {get_param: ComputeHostnameFormat}
            $TUNED_CORES: {get_param: HostIsolatedCoreList}

Add the resource mapping for the OVS-DPDK and SR-IOV services to the network-environment.yaml file along with the network configuration for these nodes:

  resource_registry:
    # Specify the relative/absolute path to the config files you want to use for override the default.
    OS::TripleO::Compute::Net::SoftwareConfig: nic-configs/compute.yaml
    OS::TripleO::Controller::Net::SoftwareConfig: nic-configs/controller.yaml
    OS::TripleO::NodeUserData: first-boot.yaml

Define the flavors:

  OvercloudControlFlavor: controller
  OvercloudComputeFlavor: compute

Define the tunnel type:

  # The tunnel type for the tenant network (vxlan or gre). Set to '' to disable tunneling.
  NeutronTunnelTypes: 'vxlan'
  # The tenant network type for Neutron (vlan or vxlan).
  NeutronNetworkType: 'vlan'

Configure the parameters for SR-IOV. You can obtain the PCI vendor and device values as seen in the NeutronSupportedPCIVendorDevs parameter by running lspci -nv.

  NeutronSupportedPCIVendorDevs: ['8086:154d', '8086:10ed']
  NovaPCIPassthrough:
    - devname: "p5p2"
      physical_network: "tenant"

  NeutronPhysicalDevMappings: "tenant:p5p2"
  NeutronSriovNumVFs: "p5p2:5"
  # Global MTU.
  NeutronGlobalPhysnetMtu: 9000
  # Configure the classname of the firewall driver to use for implementing security groups.
  NeutronOVSFirewallDriver: openvswitch

Configure the parameters for OVS-DPDK:

  ########################
  # OVS DPDK configuration
  ## NeutronDpdkCoreList and NeutronDpdkMemoryChannels are REQUIRED settings.
  ## Attempting to deploy DPDK without appropriate values will cause deployment to fail or lead to unstable deployments.
  # List of cores to be used for DPDK Poll Mode Driver
  NeutronDpdkCoreList: "'2,22,3,23'"
  # Number of memory channels to be used for DPDK
  NeutronDpdkMemoryChannels: "4"
  # NeutronDpdkSocketMemory
  NeutronDpdkSocketMemory: "'3072,1024'"
  # NeutronDpdkDriverType
  NeutronDpdkDriverType: "vfio-pci"
  # The vhost-user socket directory for OVS
  NeutronVhostuserSocketDir: "/var/lib/vhost_sockets"

  ########################
  # Additional settings
  ########################
  # Reserved RAM for host processes
  NovaReservedHostMemory: 4096
  # A list or range of physical CPU cores to reserve for virtual machine processes.
  # Example: NovaVcpuPinSet: ['4-12','^8'] will reserve cores from 4-12 excluding 8
  NovaVcpuPinSet: "4-19,24-39"
  # An array of filters used by Nova to filter a node.These filters will be applied in the order they are listed,
  # so place your most restrictive filters first to make the filtering process more efficient.
  NovaSchedulerDefaultFilters:
    - "RetryFilter"
    - "AvailabilityZoneFilter"
    - "RamFilter"
    - "ComputeFilter"
    - "ComputeCapabilitiesFilter"
    - "ImagePropertiesFilter"
    - "ServerGroupAntiAffinityFilter"
    - "ServerGroupAffinityFilter"
    - "PciPassthroughFilter"
    - "NUMATopologyFilter"
    - "AggregateInstanceExtraSpecsFilter"
  # Kernel arguments for Compute node
  ComputeKernelArgs: "default_hugepagesz=1GB hugepagesz=1G hugepages=32 iommu=pt intel_iommu=on"
  # A list or range of physical CPU cores to be tuned.
  # The given args will be appended to the tuned cpu-partitioning profile.
  HostIsolatedCoreList: "2-19,22-39"
  # List of logical cores to be used by ovs-dpdk processess (dpdk-lcore-mask)
  HostCpusList: "'2-19,22-39'"

Create the control plane Linux bond for an isolated network:

  type: linux_bond
  name: bond_api
  bonding_options: "mode=active-backup"
  use_dhcp: false
  dns_servers: {get_param: DnsServers}
  members:
    -
      type: interface
      name: nic3
      primary: true
    -
      type: interface
      name: nic4

Assign VLANs to this Linux bond:

  -
    type: vlan
    vlan_id: {get_param: InternalApiNetworkVlanID}
    device: bond_api
    addresses:
      -
        ip_netmask: {get_param: InternalApiIpSubnet}
  -
    type: vlan
    vlan_id: {get_param: StorageNetworkVlanID}
    device: bond_api
    addresses:
      -
        ip_netmask: {get_param: StorageIpSubnet}

Set a bridge with a DPDK port to link to the controller:

  type: ovs_user_bridge
  name: br-link0
  ovs_extra:
    -
      str_replace:
        template: set port br-link0 tag=_VLAN_TAG_
        params:
          _VLAN_TAG_: {get_param: TenantNetworkVlanID}
  addresses:
    -
      ip_netmask: {get_param: TenantIpSubnet}
  use_dhcp: false
  members:
    -
      type: ovs_dpdk_port
      name: dpdk0
      mtu: 9000
      ovs_extra:
      - set interface $DEVICE mtu_request=$MTU
      - set interface $DEVICE optoins:n_rxq=2
      members:
        -
          type: interface
          name: nic7
          primary: true

Create the SR-IOV interface to the Controller:

  - type: interface
    name: p7p2
    mtu: 9000
    use_dhcp: false
    defroute: false
    nm_controlled: true
    hotplug: true

Create a flavor:

# source overcloudrc
# openstack flavor create --vcpus 6 --ram 4096 --disk 40 compute

Where:

Set the flavor for large pages:

# openstack flavor set compute --property hw:mem_page_size=1GB

Create the external network:

# openstack network create --share --external \
--provider-physical-network <net-mgmt-physnet> \
--provider-network-type <flat|vlan> external

Create the networks for SR-IOV and DPDK:

# openstack network create net-dpdk
# openstack network create net-sriov
# openstack subnet create --subnet-range <cidr/prefix> --network net-dpdk  net-dpdk-subnet
# openstack subnet create --subnet-range <cidr/prefix> --network net-sriov  net-sriov-subnet

Create the SR-IOV port.

Create a router and attach to the DPDK VXLAN network:

# openstack router create router1
# openstack router add subnet router1 net-dpdk-subnet

Create a floating IP address and associate it with the guest instance port:

# openstack floating ip create --floating-ip-address FLOATING-IP external

Deploy an instance:

# openstack server create --flavor compute --image rhel_7.3 --nic port-id=sriov_port --nic net-id=NET_DPDK_ID vm1