Show Table of Contents
B.1.1.
B.1.2.
B.1.3.
B.1.4.
B.1.5.
B.1.6.
B.2.1.
B.2.2.
B.2.3.
B.2.4.
B.2.5.
B.2.6.
Appendix B. Sample OVS-DPDK YAML Files
This section provides sample OVS-DPDK YAML files as a reference.
B.1. Sample VLAN OVS-DPDK Data Plane Bonding YAML Files
B.1.1. first-boot.yaml
heat_template_version: 2014-10-16
description: >
This is an example showing how you can do firstboot configuration
of the nodes via cloud-init. To enable this, replace the default
mapping of OS::TripleO::NodeUserData in ../overcloud_resource_registry*
parameters:
ComputeKernelArgs:
description: >
Space seprated list of Kernel args to be update to grub.
The given args will be appended to existing args of GRUB_CMDLINE_LINUX in file /etc/default/grub
Example: "intel_iommu=on default_hugepagesz=1GB hugepagesz=1G hugepages=1"
type: string
default: ""
ComputeHostnameFormat:
type: string
default: ""
NeutronDpdkCoreList:
description: >
List of logical cores for PMD threads. Its mandatory parameter.
type: string
NeutronDpdkSocketMemory:
description: Memory allocated for each socket
default: ""
type: string
constraints:
- allowed_pattern: "'[0-9,]+'"
NeutronVhostuserSocketDir:
description: The vhost-user socket directory for OVS.
default: ""
type: string
HostIsolatedCoreList:
description: >
A list or range of physical CPU cores to be tuned as isolated_cores.
The given args will be appended to the tuned cpu-partitioning profile.
Ex. HostIsolatedCoreList: '4-12' will tune cores from 4-12
type: string
default: ""
HostCpusList:
description: >
List of logical cores to be used by ovs-dpdk processess (dpdk-lcore-mask)
type: string
constraints:
- allowed_pattern: "'[0-9,]+'"
resources:
userdata:
type: OS::Heat::MultipartMime
properties:
parts:
- config: {get_resource: set_dpdk_params}
- config: {get_resource: install_tuned}
- config: {get_resource: compute_kernel_args}
# Verify the logs on /var/log/cloud-init.log on the overcloud node
set_dpdk_params:
type: OS::Heat::SoftwareConfig
properties:
config:
str_replace:
template: |
#!/bin/bash
set -x
get_mask()
{
local list=$1
local mask=0
declare -a bm
max_idx=0
for core in $(echo $list | sed 's/,/ /g')
do
index=$(($core/32))
bm[$index]=0
if [ $max_idx -lt $index ]; then
max_idx=$(($index))
fi
done
for ((i=$max_idx;i>=0;i--));
do
bm[$i]=0
done
for core in $(echo $list | sed 's/,/ /g')
do
index=$(($core/32))
temp=$((1<<$(($core % 32))))
bm[$index]=$((${bm[$index]} | $temp))
done
printf -v mask "%x" "${bm[$max_idx]}"
for ((i=$max_idx-1;i>=0;i--));
do
printf -v hex "%08x" "${bm[$i]}"
mask+=$hex
done
printf "%s" "$mask"
}
FORMAT=$COMPUTE_HOSTNAME_FORMAT
if [[ -z $FORMAT ]] ; then
FORMAT="compute" ;
else
# Assumption: only %index% and %stackname% are the variables in Host name format
FORMAT=$(echo $FORMAT | sed 's/\%index\%//g' | sed 's/\%stackname\%//g') ;
fi
if [[ $(hostname) == *$FORMAT* ]] ; then
pmd_cpu_mask=$( get_mask $PMD_CORES )
host_cpu_mask=$( get_mask $LCORE_LIST )
socket_mem=$(echo $SOCKET_MEMORY | sed s/\'//g )
ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-init=true
ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-socket-mem=$socket_mem
ovs-vsctl --no-wait set Open_vSwitch . other_config:pmd-cpu-mask=$pmd_cpu_mask
ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-lcore-mask=$host_cpu_mask
fi
params:
$COMPUTE_HOSTNAME_FORMAT: {get_param: ComputeHostnameFormat}
$LCORE_LIST: {get_param: HostCpusList}
$PMD_CORES: {get_param: NeutronDpdkCoreList}
$SOCKET_MEMORY: {get_param: NeutronDpdkSocketMemory}
install_tuned:
type: OS::Heat::SoftwareConfig
properties:
config:
str_replace:
template: |
#!/bin/bash
FORMAT=$COMPUTE_HOSTNAME_FORMAT
if [[ -z $FORMAT ]] ; then
FORMAT="compute" ;
else
# Assumption: only %index% and %stackname% are the variables in Host name format
FORMAT=$(echo $FORMAT | sed 's/\%index\%//g' | sed 's/\%stackname\%//g') ;
fi
if [[ $(hostname) == *$FORMAT* ]] ; then
# Install the tuned package
yum install -y tuned-profiles-cpu-partitioning
tuned_conf_path="/etc/tuned/cpu-partitioning-variables.conf"
if [ -n "$TUNED_CORES" ]; then
grep -q "^isolated_cores" $tuned_conf_path
if [ "$?" -eq 0 ]; then
sed -i 's/^isolated_cores=.*/isolated_cores=$TUNED_CORES/' $tuned_conf_path
else
echo "isolated_cores=$TUNED_CORES" >> $tuned_conf_path
fi
tuned-adm profile cpu-partitioning
fi
fi
params:
$COMPUTE_HOSTNAME_FORMAT: {get_param: ComputeHostnameFormat}
$TUNED_CORES: {get_param: HostIsolatedCoreList}
compute_kernel_args:
type: OS::Heat::SoftwareConfig
properties:
config:
str_replace:
template: |
#!/bin/bash
FORMAT=$COMPUTE_HOSTNAME_FORMAT
if [[ -z $FORMAT ]] ; then
FORMAT="compute" ;
else
# Assumption: only %index% and %stackname% are the variables in Host name format
FORMAT=$(echo $FORMAT | sed 's/\%index\%//g' | sed 's/\%stackname\%//g') ;
fi
if [[ $(hostname) == *$FORMAT* ]] ; then
sed 's/^\(GRUB_CMDLINE_LINUX=".*\)"/\1 $KERNEL_ARGS isolcpus=$TUNED_CORES"/g' -i /etc/default/grub ;
grub2-mkconfig -o /etc/grub2.cfg
reboot
fi
params:
$KERNEL_ARGS: {get_param: ComputeKernelArgs}
$COMPUTE_HOSTNAME_FORMAT: {get_param: ComputeHostnameFormat}
$TUNED_CORES: {get_param: HostIsolatedCoreList}
outputs:
# This means get_resource from the parent template will get the userdata, see:
# http://docs.openstack.org/developer/heat/template_guide/composition.html#making-your-template-resource-more-transparent
# Note this is new-for-kilo, an alternative is returning a value then using
# get_attr in the parent template instead.
OS::stack_id:
value: {get_resource: userdata}B.1.2. post-install.yaml
heat_template_version: 2014-10-16
description: >
Example extra config for post-deployment
parameters:
servers:
type: json
ComputeHostnameFormat:
type: string
default: ""
resources:
ExtraDeployments:
type: OS::Heat::StructuredDeployments
properties:
servers: {get_param: servers}
config: {get_resource: ExtraConfig}
# Do this on CREATE/UPDATE (which is actually the default)
actions: ['CREATE', 'UPDATE']
ExtraConfig:
type: OS::Heat::SoftwareConfig
properties:
group: script
config:
str_replace:
template: |
#!/bin/bash
set -x
FORMAT=$COMPUTE_HOSTNAME_FORMAT
if [[ -z $FORMAT ]] ; then
FORMAT="compute" ;
else
# Assumption: only %index% and %stackname% are the variables in Host name format
FORMAT=$(echo $FORMAT | sed 's/\%index\%//g' | sed 's/\%stackname\%//g') ;
fi
if [[ $(hostname) == *$FORMAT* ]] ; then
tuned_service=/usr/lib/systemd/system/tuned.service
grep -q "network.target" $tuned_service
if [ "$?" -eq 0 ]; then
sed -i '/After=.*/s/network.target//g' $tuned_service
fi
grep -q "Before=.*network.target" $tuned_service
if [ ! "$?" -eq 0 ]; then
grep -q "Before=.*" $tuned_service
if [ "$?" -eq 0 ]; then
sed -i 's/^\(Before=.*\)/\1 network.target openvswitch.service/g' $tuned_service
else
sed -i '/After/i Before=network.target openvswitch.service' $tuned_service
fi
fi
systemctl daemon-reload
fi
params:
$COMPUTE_HOSTNAME_FORMAT: {get_param: ComputeHostnameFormat}B.1.3. network.environment.yaml
resource_registry:
# Specify the relative/absolute path to the config files you want to use for override the default.
OS::TripleO::Compute::Net::SoftwareConfig: nic-configs/compute.yaml
OS::TripleO::Controller::Net::SoftwareConfig: nic-configs/controller.yaml
OS::TripleO::NodeUserData: first-boot.yaml
OS::TripleO::NodeExtraConfigPost: post-install.yaml
parameter_defaults:
# Customize all these values to match the local environment
InternalApiNetCidr: 10.10.112.0/24
TenantNetCidr: 10.10.113.0/24
StorageNetCidr: 10.10.114.0/24
StorageMgmtNetCidr: 10.10.115.0/24
ExternalNetCidr: 10.35.141.64/28
# CIDR subnet mask length for provisioning network
ControlPlaneSubnetCidr: '24'
InternalApiAllocationPools: [{'start': '10.10.112.100', 'end': '10.10.112.200'}]
TenantAllocationPools: [{'start': '10.10.113.100', 'end': '10.10.113.200'}]
StorageAllocationPools: [{'start': '10.10.114.100', 'end': '10.10.114.200'}]
StorageMgmtAllocationPools: [{'start': '10.10.115.100', 'end': '10.10.115.200'}]
# Use an External allocation pool which will leave room for floating IPs
ExternalAllocationPools: [{'start': '10.35.141.67', 'end': '10.35.141.70'}]
# Set to the router gateway on the external network
ExternalInterfaceDefaultRoute: 10.35.141.78
# Gateway router for the provisioning network (or Undercloud IP)
ControlPlaneDefaultRoute: 192.0.90.1
# Generally the IP of the Undercloud
EC2MetadataIp: 192.0.90.1
InternalApiNetworkVlanID: 512
TenantNetworkVlanID: 513
StorageNetworkVlanID: 514
StorageMgmtNetworkVlanID: 515
ExternalNetworkVlanID: 400
# Define the DNS servers (maximum 2) for the overcloud nodes
DnsServers: ["10.35.28.1","10.35.28.28"]
# May set to br-ex if using floating IPs only on native VLAN on bridge br-ex
NeutronExternalNetworkBridge: "''"
# The tunnel type for the tenant network (vxlan or gre). Set to '' to disable tunneling.
NeutronTunnelTypes: ''
# The tenant network type for Neutron (vlan or vxlan).
NeutronNetworkType: 'vlan'
# The OVS logical->physical bridge mappings to use.
NeutronBridgeMappings: 'tenant:br-link0'
# The Neutron ML2 and OpenVSwitch vlan mapping range to support.
NeutronNetworkVLANRanges: 'tenant:400:400,tenant:422:422,tenant:424:424'
# Nova flavor to use.
OvercloudControlFlavor: controller
OvercloudComputeFlavor: compute
#Number of nodes to deploy.
ControllerCount: 1
ComputeCount: 1
# NTP server configuration.
NtpServer: clock.redhat.com
# Sets overcloud nodes custom names
# http://docs.openstack.org/developer/tripleo-docs/advanced_deployment/node_placement.html#custom-hostnames
ControllerHostnameFormat: 'controller-%index%'
ComputeHostnameFormat: 'compute-%index%'
CephStorageHostnameFormat: 'ceph-%index%'
ObjectStorageHostnameFormat: 'swift-%index%'
########################
# OVS DPDK configuration
## NeutronDpdkCoreList and NeutronDpdkMemoryChannels are REQUIRED settings.
## Attempting to deploy DPDK without appropriate values will cause deployment to fail or lead to unstable deployments.
# List of cores to be used for DPDK Poll Mode Driver
NeutronDpdkCoreList: "'1,17,9,25'"
# Number of memory channels to be used for DPDK
NeutronDpdkMemoryChannels: "4"
# NeutronDpdkSocketMemory
NeutronDpdkSocketMemory: "'1024,1024'"
# NeutronDpdkDriverType
NeutronDpdkDriverType: "vfio-pci"
# The vhost-user socket directory for OVS
NeutronVhostuserSocketDir: "/var/lib/vhost_sockets"
########################
# Additional settings
########################
# Reserved RAM for host processes
NovaReservedHostMemory: 2048
# A list or range of physical CPU cores to reserve for virtual machine processes.
# Example: NovaVcpuPinSet: ['4-12','^8'] will reserve cores from 4-12 excluding 8
NovaVcpuPinSet: "2,3,4,5,6,7,18,19,20,21,22,23,10,11,12,13,14,15,26,27,28,29,30,31"
# An array of filters used by Nova to filter a node.These filters will be applied in the order they are listed,
# so place your most restrictive filters first to make the filtering process more efficient.
NovaSchedulerDefaultFilters: "RamFilter,ComputeFilter,AvailabilityZoneFilter,ComputeCapabilitiesFilter,ImagePropertiesFilter,PciPassthroughFilter,NUMATopologyFilter"
# Kernel arguments for Compute node
ComputeKernelArgs: "default_hugepagesz=1GB hugepagesz=1G hugepages=32 iommu=pt intel_iommu=on"
# A list or range of physical CPU cores to be tuned.
# The given args will be appended to the tuned cpu-partitioning profile.
HostIsolatedCoreList: "1,2,3,4,5,6,7,9,10,17,18,19,20,21,22,23,11,12,13,14,15,25,26,27,28,29,30,31"
# List of logical cores to be used by ovs-dpdk processess (dpdk-lcore-mask)
HostCpusList: "'0,16,8,24'"
# MTU global configuration
NeutronGlobalPhysnetMtu: 9000
# DHCP provide metadata route to VM.
NeutronEnableIsolatedMetadata: true
# DHCP always provides metadata route to VM.
NeutronEnableForceMetadata: true
# Set the storage backend of the overcloud
GlanceBackend: 'file'
# Configure the classname of the firewall driver to use for implementing security groups.
NeutronOVSFirewallDriver: openvswitch
SshServerOptions:
UseDns: 'no'B.1.4. controller.yaml
heat_template_version: 2015-04-30
description: >
Software Config to drive os-net-config to configure VLANs for the
controller role.
parameters:
ControlPlaneIp:
default: ''
description: IP address/subnet on the ctlplane network
type: string
ExternalIpSubnet:
default: ''
description: IP address/subnet on the external network
type: string
InternalApiIpSubnet:
default: ''
description: IP address/subnet on the internal API network
type: string
StorageIpSubnet:
default: ''
description: IP address/subnet on the storage network
type: string
StorageMgmtIpSubnet:
default: ''
description: IP address/subnet on the storage mgmt network
type: string
StorageNetworkVlanID:
default: 30
description: Vlan ID for the storage network traffic.
type: number
StorageMgmtNetworkVlanID:
default: 40
description: Vlan ID for the storage mgmt network traffic.
type: number
TenantIpSubnet:
default: ''
description: IP address/subnet on the tenant network
type: string
ManagementIpSubnet: # Only populated when including environments/network-management.yaml
default: ''
description: IP address/subnet on the management network
type: string
ExternalNetworkVlanID:
default: ''
description: Vlan ID for the external network traffic.
type: number
InternalApiNetworkVlanID:
default: ''
description: Vlan ID for the internal_api network traffic.
type: number
TenantNetworkVlanID:
default: ''
description: Vlan ID for the tenant network traffic.
type: number
ManagementNetworkVlanID:
default: 23
description: Vlan ID for the management network traffic.
type: number
ExternalInterfaceDefaultRoute:
default: ''
description: default route for the external network
type: string
ControlPlaneSubnetCidr: # Override this via parameter_defaults
default: '24'
description: The subnet CIDR of the control plane network.
type: string
DnsServers: # Override this via parameter_defaults
default: []
description: A list of DNS servers (2 max for some implementations) that will be added to resolv.conf.
type: comma_delimited_list
EC2MetadataIp: # Override this via parameter_defaults
description: The IP address of the EC2 metadata server.
type: string
resources:
OsNetConfigImpl:
type: OS::Heat::StructuredConfig
properties:
group: os-apply-config
config:
os_net_config:
network_config:
-
type: interface
name: nic1
use_dhcp: false
defroute: false
-
type: interface
name: nic2
addresses:
-
ip_netmask:
list_join:
- '/'
- - {get_param: ControlPlaneIp}
- {get_param: ControlPlaneSubnetCidr}
routes:
-
ip_netmask: 169.254.169.254/32
next_hop: {get_param: EC2MetadataIp}
-
type: linux_bond
name: bond_api
bonding_options: "mode=active-backup"
use_dhcp: false
dns_servers: {get_param: DnsServers}
members:
-
type: interface
name: nic7
primary: true
-
type: interface
name: nic8
-
type: vlan
vlan_id: {get_param: InternalApiNetworkVlanID}
device: bond_api
addresses:
-
ip_netmask: {get_param: InternalApiIpSubnet}
-
type: vlan
vlan_id: {get_param: TenantNetworkVlanID}
device: bond_api
addresses:
-
ip_netmask: {get_param: TenantIpSubnet}
-
type: vlan
vlan_id: {get_param: StorageNetworkVlanID}
device: bond_api
addresses:
-
ip_netmask: {get_param: StorageIpSubnet}
-
type: vlan
vlan_id: {get_param: StorageMgmtNetworkVlanID}
device: bond_api
addresses:
-
ip_netmask: {get_param: StorageMgmtIpSubnet}
-
type: vlan
vlan_id: {get_param: ExternalNetworkVlanID}
device: bond_api
addresses:
-
ip_netmask: {get_param: ExternalIpSubnet}
routes:
-
default: true
next_hop: {get_param: ExternalInterfaceDefaultRoute}
-
type: ovs_bridge
name: br-link0
use_dhcp: false
mtu: 9000
members:
-
type: ovs_bond
name: bond0
use_dhcp: true
members:
-
type: interface
name: nic3
mtu: 9000
-
type: interface
name: nic4
mtu: 9000
outputs:
OS::stack_id:
description: The OsNetConfigImpl resource.
value: {get_resource: OsNetConfigImpl}B.1.5. compute-ovs-dpdk.yaml
heat_template_version: 2015-04-30
description: >
Software Config to drive os-net-config to configure VLANs for the
compute role.
parameters:
ControlPlaneIp:
default: ''
description: IP address/subnet on the ctlplane network
type: string
ExternalIpSubnet:
default: ''
description: IP address/subnet on the external network
type: string
InternalApiIpSubnet:
default: ''
description: IP address/subnet on the internal API network
type: string
TenantIpSubnet:
default: ''
description: IP address/subnet on the tenant network
type: string
ManagementIpSubnet: # Only populated when including environments/network-management.yaml
default: ''
description: IP address/subnet on the management network
type: string
InternalApiNetworkVlanID:
default: ''
description: Vlan ID for the internal_api network traffic.
type: number
TenantNetworkVlanID:
default: ''
description: Vlan ID for the tenant network traffic.
type: number
ManagementNetworkVlanID:
default: 23
description: Vlan ID for the management network traffic.
type: number
StorageIpSubnet:
default: ''
description: IP address/subnet on the storage network
type: string
StorageMgmtIpSubnet:
default: ''
description: IP address/subnet on the storage mgmt network
type: string
StorageNetworkVlanID:
default: 30
description: Vlan ID for the storage network traffic.
type: number
StorageMgmtNetworkVlanID:
default: 40
description: Vlan ID for the storage mgmt network traffic.
type: number
ControlPlaneSubnetCidr: # Override this via parameter_defaults
default: '24'
description: The subnet CIDR of the control plane network.
type: string
ControlPlaneDefaultRoute: # Override this via parameter_defaults
description: The default route of the control plane network.
type: string
DnsServers: # Override this via parameter_defaults
default: []
description: A list of DNS servers (2 max for some implementations) that will be added to resolv.conf.
type: comma_delimited_list
EC2MetadataIp: # Override this via parameter_defaults
description: The IP address of the EC2 metadata server.
type: string
ExternalInterfaceDefaultRoute:
default: ''
description: default route for the external network
type: string
resources:
OsNetConfigImpl:
type: OS::Heat::StructuredConfig
properties:
group: os-apply-config
config:
os_net_config:
network_config:
-
type: interface
name: nic1
use_dhcp: false
addresses:
-
ip_netmask:
list_join:
- '/'
- - {get_param: ControlPlaneIp}
- {get_param: ControlPlaneSubnetCidr}
routes:
-
ip_netmask: 169.254.169.254/32
next_hop: {get_param: EC2MetadataIp}
-
default: true
next_hop: {get_param: ControlPlaneDefaultRoute}
-
type: linux_bond
name: bond_api
bonding_options: "mode=active-backup"
use_dhcp: false
dns_servers: {get_param: DnsServers}
members:
-
type: interface
name: nic2
primary: true
-
type: interface
name: nic3
-
type: vlan
vlan_id: {get_param: InternalApiNetworkVlanID}
device: bond_api
addresses:
-
ip_netmask: {get_param: InternalApiIpSubnet}
-
type: vlan
vlan_id: {get_param: TenantNetworkVlanID}
device: bond_api
addresses:
-
ip_netmask: {get_param: TenantIpSubnet}
-
type: vlan
vlan_id: {get_param: StorageNetworkVlanID}
device: bond_api
addresses:
-
ip_netmask: {get_param: StorageIpSubnet}
-
type: ovs_user_bridge
name: br-link
use_dhcp: false
members:
-
type: ovs_dpdk_bond
name: dpdkbond0
members:
-
type: ovs_dpdk_port
name: dpdk0
members:
-
type: interface
name: nic4
-
type: ovs_dpdk_port
name: dpdk1
members:
-
type: interface
name: nic5
outputs:
OS::stack_id:
description: The OsNetConfigImpl resource.
value: {get_resource: OsNetConfigImpl}B.1.6. overcloud_deploy.sh
#!/bin/bash openstack overcloud deploy \ --templates \ -e /usr/share/openstack-tripleo-heat-templates/environments/network-isolation.yaml \ -e /usr/share/openstack-tripleo-heat-templates/environments/neutron-ovs-dpdk.yaml \ -e /usr/share/openstack-tripleo-heat-templates/environments/ovs-dpdk-permissions.yaml \ -e /home/stack/ospd-10-vlan-dpdk-two-ports-ctlplane-dataplane-bonding/network-environment.yaml \ --log-file overcloud_install.log &> overcloud_install.log
B.2. Sample VXLAN OVS-DPDK Data Plane Bonding YAML Files
B.2.1. first-boot.yaml
heat_template_version: 2014-10-16
description: >
This is an example showing how you can do firstboot configuration
of the nodes via cloud-init. To enable this, replace the default
mapping of OS::TripleO::NodeUserData in ../overcloud_resource_registry*
parameters:
ComputeKernelArgs:
description: >
Space seprated list of Kernel args to be update to grub.
The given args will be appended to existing args of GRUB_CMDLINE_LINUX in file /etc/default/grub
Example: "intel_iommu=on default_hugepagesz=1GB hugepagesz=1G hugepages=1"
type: string
default: ""
ComputeHostnameFormat:
type: string
default: ""
NeutronDpdkCoreList:
description: >
List of logical cores for PMD threads. Its mandatory parameter.
type: string
NeutronDpdkSocketMemory:
description: Memory allocated for each socket
default: ""
type: string
constraints:
- allowed_pattern: "'[0-9,]+'"
NeutronVhostuserSocketDir:
description: The vhost-user socket directory for OVS.
default: ""
type: string
HostIsolatedCoreList:
description: >
A list or range of physical CPU cores to be tuned as isolated_cores.
The given args will be appended to the tuned cpu-partitioning profile.
Ex. HostIsolatedCoreList: '4-12' will tune cores from 4-12
type: string
default: ""
HostCpusList:
description: >
List of logical cores to be used by ovs-dpdk processess (dpdk-lcore-mask)
type: string
constraints:
- allowed_pattern: "'[0-9,]+'"
resources:
userdata:
type: OS::Heat::MultipartMime
properties:
parts:
- config: {get_resource: set_dpdk_params}
- config: {get_resource: install_tuned}
- config: {get_resource: compute_kernel_args}
# Verify the logs on /var/log/cloud-init.log on the overcloud node
set_dpdk_params:
type: OS::Heat::SoftwareConfig
properties:
config:
str_replace:
template: |
#!/bin/bash
set -x
get_mask()
{
local list=$1
local mask=0
declare -a bm
max_idx=0
for core in $(echo $list | sed 's/,/ /g')
do
index=$(($core/32))
bm[$index]=0
if [ $max_idx -lt $index ]; then
max_idx=$(($index))
fi
done
for ((i=$max_idx;i>=0;i--));
do
bm[$i]=0
done
for core in $(echo $list | sed 's/,/ /g')
do
index=$(($core/32))
temp=$((1<<$(($core % 32))))
bm[$index]=$((${bm[$index]} | $temp))
done
printf -v mask "%x" "${bm[$max_idx]}"
for ((i=$max_idx-1;i>=0;i--));
do
printf -v hex "%08x" "${bm[$i]}"
mask+=$hex
done
printf "%s" "$mask"
}
FORMAT=$COMPUTE_HOSTNAME_FORMAT
if [[ -z $FORMAT ]] ; then
FORMAT="compute" ;
else
# Assumption: only %index% and %stackname% are the variables in Host name format
FORMAT=$(echo $FORMAT | sed 's/\%index\%//g' | sed 's/\%stackname\%//g') ;
fi
if [[ $(hostname) == *$FORMAT* ]] ; then
pmd_cpu_mask=$( get_mask $PMD_CORES )
host_cpu_mask=$( get_mask $LCORE_LIST )
socket_mem=$(echo $SOCKET_MEMORY | sed s/\'//g )
ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-init=true
ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-socket-mem=$socket_mem
ovs-vsctl --no-wait set Open_vSwitch . other_config:pmd-cpu-mask=$pmd_cpu_mask
ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-lcore-mask=$host_cpu_mask
fi
params:
$COMPUTE_HOSTNAME_FORMAT: {get_param: ComputeHostnameFormat}
$LCORE_LIST: {get_param: HostCpusList}
$PMD_CORES: {get_param: NeutronDpdkCoreList}
$SOCKET_MEMORY: {get_param: NeutronDpdkSocketMemory}
install_tuned:
type: OS::Heat::SoftwareConfig
properties:
config:
str_replace:
template: |
#!/bin/bash
FORMAT=$COMPUTE_HOSTNAME_FORMAT
if [[ -z $FORMAT ]] ; then
FORMAT="compute" ;
else
# Assumption: only %index% and %stackname% are the variables in Host name format
FORMAT=$(echo $FORMAT | sed 's/\%index\%//g' | sed 's/\%stackname\%//g') ;
fi
if [[ $(hostname) == *$FORMAT* ]] ; then
# Install the tuned package
yum install -y tuned-profiles-cpu-partitioning
tuned_conf_path="/etc/tuned/cpu-partitioning-variables.conf"
if [ -n "$TUNED_CORES" ]; then
grep -q "^isolated_cores" $tuned_conf_path
if [ "$?" -eq 0 ]; then
sed -i 's/^isolated_cores=.*/isolated_cores=$TUNED_CORES/' $tuned_conf_path
else
echo "isolated_cores=$TUNED_CORES" >> $tuned_conf_path
fi
tuned-adm profile cpu-partitioning
fi
fi
params:
$COMPUTE_HOSTNAME_FORMAT: {get_param: ComputeHostnameFormat}
$TUNED_CORES: {get_param: HostIsolatedCoreList}
compute_kernel_args:
type: OS::Heat::SoftwareConfig
properties:
config:
str_replace:
template: |
#!/bin/bash
FORMAT=$COMPUTE_HOSTNAME_FORMAT
if [[ -z $FORMAT ]] ; then
FORMAT="compute" ;
else
# Assumption: only %index% and %stackname% are the variables in Host name format
FORMAT=$(echo $FORMAT | sed 's/\%index\%//g' | sed 's/\%stackname\%//g') ;
fi
if [[ $(hostname) == *$FORMAT* ]] ; then
sed 's/^\(GRUB_CMDLINE_LINUX=".*\)"/\1 $KERNEL_ARGS isolcpus=$TUNED_CORES"/g' -i /etc/default/grub ;
grub2-mkconfig -o /etc/grub2.cfg
reboot
fi
params:
$KERNEL_ARGS: {get_param: ComputeKernelArgs}
$COMPUTE_HOSTNAME_FORMAT: {get_param: ComputeHostnameFormat}
$TUNED_CORES: {get_param: HostIsolatedCoreList}
outputs:
# This means get_resource from the parent template will get the userdata, see:
# http://docs.openstack.org/developer/heat/template_guide/composition.html#making-your-template-resource-more-transparent
# Note this is new-for-kilo, an alternative is returning a value then using
# get_attr in the parent template instead.
OS::stack_id:
value: {get_resource: userdata}B.2.2. post-install.yaml
heat_template_version: 2014-10-16
description: >
Example extra config for post-deployment
parameters:
servers:
type: json
ComputeHostnameFormat:
type: string
default: ""
resources:
ExtraDeployments:
type: OS::Heat::StructuredDeployments
properties:
servers: {get_param: servers}
config: {get_resource: ExtraConfig}
# Do this on CREATE/UPDATE (which is actually the default)
actions: ['CREATE', 'UPDATE']
ExtraConfig:
type: OS::Heat::SoftwareConfig
properties:
group: script
config:
str_replace:
template: |
#!/bin/bash
set -x
FORMAT=$COMPUTE_HOSTNAME_FORMAT
if [[ -z $FORMAT ]] ; then
FORMAT="compute" ;
else
# Assumption: only %index% and %stackname% are the variables in Host name format
FORMAT=$(echo $FORMAT | sed 's/\%index\%//g' | sed 's/\%stackname\%//g') ;
fi
if [[ $(hostname) == *$FORMAT* ]] ; then
tuned_service=/usr/lib/systemd/system/tuned.service
grep -q "network.target" $tuned_service
if [ "$?" -eq 0 ]; then
sed -i '/After=.*/s/network.target//g' $tuned_service
fi
grep -q "Before=.*network.target" $tuned_service
if [ ! "$?" -eq 0 ]; then
grep -q "Before=.*" $tuned_service
if [ "$?" -eq 0 ]; then
sed -i 's/^\(Before=.*\)/\1 network.target openvswitch.service/g' $tuned_service
else
sed -i '/After/i Before=network.target openvswitch.service' $tuned_service
fi
fi
systemctl daemon-reload
fi
params:
$COMPUTE_HOSTNAME_FORMAT: {get_param: ComputeHostnameFormat}B.2.3. network.environment.yaml
resource_registry:
# Specify the relative/absolute path to the config files you want to use for override the default.
OS::TripleO::Compute::Net::SoftwareConfig: nic-configs/compute.yaml
OS::TripleO::Controller::Net::SoftwareConfig: nic-configs/controller.yaml
OS::TripleO::NodeUserData: first-boot.yaml
OS::TripleO::NodeExtraConfigPost: post-install.yaml
OS::TripleO::AllNodes::Validation: dummy_all_nodes-validation.yaml
parameter_defaults:
# Customize all these values to match the local environment
InternalApiNetCidr: 10.10.112.0/24
TenantNetCidr: 10.10.113.0/24
StorageNetCidr: 10.10.114.0/24
StorageMgmtNetCidr: 10.10.115.0/24
ExternalNetCidr: 10.35.141.64/28
# CIDR subnet mask length for provisioning network
ControlPlaneSubnetCidr: '24'
InternalApiAllocationPools: [{'start': '10.10.112.100', 'end': '10.10.112.200'}]
TenantAllocationPools: [{'start': '10.10.113.100', 'end': '10.10.113.200'}]
StorageAllocationPools: [{'start': '10.10.114.100', 'end': '10.10.114.200'}]
StorageMgmtAllocationPools: [{'start': '10.10.115.100', 'end': '10.10.115.200'}]
# Use an External allocation pool which will leave room for floating IPs
ExternalAllocationPools: [{'start': '10.35.141.67', 'end': '10.35.141.70'}]
# Set to the router gateway on the external network
ExternalInterfaceDefaultRoute: 10.35.141.78
# Gateway router for the provisioning network (or Undercloud IP)
ControlPlaneDefaultRoute: 192.0.90.1
# Generally the IP of the Undercloud
EC2MetadataIp: 192.0.90.1
InternalApiNetworkVlanID: 512
TenantNetworkVlanID: 513
StorageNetworkVlanID: 514
StorageMgmtNetworkVlanID: 515
ExternalNetworkVlanID: 400
# Define the DNS servers (maximum 2) for the overcloud nodes
DnsServers: ["10.35.28.1","10.35.28.28"]
# May set to br-ex if using floating IPs only on native VLAN on bridge br-ex
NeutronExternalNetworkBridge: "''"
# The tunnel type for the tenant network (vxlan or gre). Set to '' to disable tunneling.
NeutronTunnelTypes: 'vxlan'
# The tenant network type for Neutron (vlan or vxlan).
NeutronNetworkType: 'vxlan'
# The OVS logical->physical bridge mappings to use.
NeutronBridgeMappings: 'tenant:br-link0'
# The Neutron ML2 and OpenVSwitch vlan mapping range to support.
NeutronNetworkVLANRanges: 'tenant:400:400'
# Nova flavor to use.
OvercloudControlFlavor: controller
OvercloudComputeFlavor: compute
#Number of nodes to deploy.
ControllerCount: 1
ComputeCount: 1
# NTP server configuration.
NtpServer: clock.redhat.com
# Sets overcloud nodes custom names
# http://docs.openstack.org/developer/tripleo-docs/advanced_deployment/node_placement.html#custom-hostnames
ControllerHostnameFormat: 'controller-%index%'
ComputeHostnameFormat: 'compute-%index%'
CephStorageHostnameFormat: 'ceph-%index%'
ObjectStorageHostnameFormat: 'swift-%index%'
########################
# OVS DPDK configuration
## NeutronDpdkCoreList and NeutronDpdkMemoryChannels are REQUIRED settings.
## Attempting to deploy DPDK without appropriate values will cause deployment to fail or lead to unstable deployments.
# List of cores to be used for DPDK Poll Mode Driver
NeutronDpdkCoreList: "'1,17,9,25'"
# Number of memory channels to be used for DPDK
NeutronDpdkMemoryChannels: "4"
# NeutronDpdkSocketMemory
NeutronDpdkSocketMemory: "'1024,1024'"
# NeutronDpdkDriverType
NeutronDpdkDriverType: "vfio-pci"
# The vhost-user socket directory for OVS
NeutronVhostuserSocketDir: "/var/lib/vhost_sockets"
########################
# Additional settings
########################
# Reserved RAM for host processes
NovaReservedHostMemory: 2048
# A list or range of physical CPU cores to reserve for virtual machine processes.
# Example: NovaVcpuPinSet: ['4-12','^8'] will reserve cores from 4-12 excluding 8
NovaVcpuPinSet: "2,3,4,5,6,7,18,19,20,21,22,23,10,11,12,13,14,15,26,27,28,29,30,31"
# An array of filters used by Nova to filter a node.These filters will be applied in the order they are listed,
# so place your most restrictive filters first to make the filtering process more efficient.
NovaSchedulerDefaultFilters: "RamFilter,ComputeFilter,AvailabilityZoneFilter,ComputeCapabilitiesFilter,ImagePropertiesFilter,PciPassthroughFilter,NUMATopologyFilter"
# Kernel arguments for Compute node
ComputeKernelArgs: "default_hugepagesz=1GB hugepagesz=1G hugepages=32 iommu=pt intel_iommu=on"
# A list or range of physical CPU cores to be tuned.
# The given args will be appended to the tuned cpu-partitioning profile.
HostIsolatedCoreList: "1,2,3,4,5,6,7,9,10,17,18,19,20,21,22,23,11,12,13,14,15,25,26,27,28,29,30,31"
# List of logical cores to be used by ovs-dpdk processess (dpdk-lcore-mask)
HostCpusList: "'0,16,8,24'"
# MTU global configuration
NeutronGlobalPhysnetMtu: 9000
# DHCP provide metadata route to VM.
NeutronEnableIsolatedMetadata: true
# DHCP always provides metadata route to VM.
NeutronEnableForceMetadata: true
# Set the storage backend of the overcloud
GlanceBackend: 'file'
# Configure the classname of the firewall driver to use for implementing security groups.
NeutronOVSFirewallDriver: openvswitch
SshServerOptions:
UseDns: 'no'B.2.4. controller.yaml
heat_template_version: 2015-04-30
description: >
Software Config to drive os-net-config to configure VLANs for the
controller role.
parameters:
ControlPlaneIp:
default: ''
description: IP address/subnet on the ctlplane network
type: string
ExternalIpSubnet:
default: ''
description: IP address/subnet on the external network
type: string
InternalApiIpSubnet:
default: ''
description: IP address/subnet on the internal API network
type: string
StorageIpSubnet:
default: ''
description: IP address/subnet on the storage network
type: string
StorageMgmtIpSubnet:
default: ''
description: IP address/subnet on the storage mgmt network
type: string
StorageNetworkVlanID:
default: 30
description: Vlan ID for the storage network traffic.
type: number
StorageMgmtNetworkVlanID:
default: 40
description: Vlan ID for the storage mgmt network traffic.
type: number
TenantIpSubnet:
default: ''
description: IP address/subnet on the tenant network
type: string
ManagementIpSubnet: # Only populated when including environments/network-management.yaml
default: ''
description: IP address/subnet on the management network
type: string
ExternalNetworkVlanID:
default: ''
description: Vlan ID for the external network traffic.
type: number
InternalApiNetworkVlanID:
default: ''
description: Vlan ID for the internal_api network traffic.
type: number
TenantNetworkVlanID:
default: ''
description: Vlan ID for the tenant network traffic.
type: number
ManagementNetworkVlanID:
default: 23
description: Vlan ID for the management network traffic.
type: number
ExternalInterfaceDefaultRoute:
default: ''
description: default route for the external network
type: string
ControlPlaneSubnetCidr: # Override this via parameter_defaults
default: '24'
description: The subnet CIDR of the control plane network.
type: string
DnsServers: # Override this via parameter_defaults
default: []
description: A list of DNS servers (2 max for some implementations) that will be added to resolv.conf.
type: comma_delimited_list
EC2MetadataIp: # Override this via parameter_defaults
description: The IP address of the EC2 metadata server.
type: string
resources:
OsNetConfigImpl:
type: OS::Heat::StructuredConfig
properties:
group: os-apply-config
config:
os_net_config:
network_config:
-
type: interface
name: nic1
use_dhcp: false
defroute: false
-
type: interface
name: nic2
addresses:
-
ip_netmask:
list_join:
- '/'
- - {get_param: ControlPlaneIp}
- {get_param: ControlPlaneSubnetCidr}
routes:
-
ip_netmask: 169.254.169.254/32
next_hop: {get_param: EC2MetadataIp}
-
type: linux_bond
name: bond_api
bonding_options: "mode=active-backup"
use_dhcp: false
dns_servers: {get_param: DnsServers}
members:
-
type: interface
name: nic7
primary: true
-
type: interface
name: nic8
-
type: vlan
vlan_id: {get_param: InternalApiNetworkVlanID}
device: bond_api
addresses:
-
ip_netmask: {get_param: InternalApiIpSubnet}
-
type: vlan
vlan_id: {get_param: StorageNetworkVlanID}
device: bond_api
addresses:
-
ip_netmask: {get_param: StorageIpSubnet}
-
type: vlan
vlan_id: {get_param: StorageMgmtNetworkVlanID}
device: bond_api
addresses:
-
ip_netmask: {get_param: StorageMgmtIpSubnet}
-
type: vlan
vlan_id: {get_param: ExternalNetworkVlanID}
device: bond_api
addresses:
-
ip_netmask: {get_param: ExternalIpSubnet}
routes:
-
default: true
next_hop: {get_param: ExternalInterfaceDefaultRoute}
-
type: ovs_bridge
name: br-link0
use_dhcp: false
mtu: 9000
members:
-
type: ovs_bond
name: bond0
use_dhcp: true
members:
-
type: interface
name: nic3
mtu: 9000
-
type: interface
name: nic4
mtu: 9000
-
type: vlan
vlan_id: {get_param: TenantNetworkVlanID}
device: bond0
mtu: 9000
addresses:
-
ip_netmask: {get_param: TenantIpSubnet}
outputs:
OS::stack_id:
description: The OsNetConfigImpl resource.
value: {get_resource: OsNetConfigImpl}B.2.5. compute-ovs-dpdk.yaml
heat_template_version: 2015-04-30
description: >
Software Config to drive os-net-config to configure VLANs for the
compute role.
parameters:
ControlPlaneIp:
default: ''
description: IP address/subnet on the ctlplane network
type: string
ExternalIpSubnet:
default: ''
description: IP address/subnet on the external network
type: string
InternalApiIpSubnet:
default: ''
description: IP address/subnet on the internal API network
type: string
TenantIpSubnet:
default: ''
description: IP address/subnet on the tenant network
type: string
ManagementIpSubnet: # Only populated when including environments/network-management.yaml
default: ''
description: IP address/subnet on the management network
type: string
InternalApiNetworkVlanID:
default: ''
description: Vlan ID for the internal_api network traffic.
type: number
TenantNetworkVlanID:
default: ''
description: Vlan ID for the tenant network traffic.
type: number
ManagementNetworkVlanID:
default: 23
description: Vlan ID for the management network traffic.
type: number
StorageIpSubnet:
default: ''
description: IP address/subnet on the storage network
type: string
StorageMgmtIpSubnet:
default: ''
description: IP address/subnet on the storage mgmt network
type: string
StorageNetworkVlanID:
default: 30
description: Vlan ID for the storage network traffic.
type: number
StorageMgmtNetworkVlanID:
default: 40
description: Vlan ID for the storage mgmt network traffic.
type: number
ControlPlaneSubnetCidr: # Override this via parameter_defaults
default: '24'
description: The subnet CIDR of the control plane network.
type: string
ControlPlaneDefaultRoute: # Override this via parameter_defaults
description: The default route of the control plane network.
type: string
DnsServers: # Override this via parameter_defaults
default: []
description: A list of DNS servers (2 max for some implementations) that will be added to resolv.conf.
type: comma_delimited_list
EC2MetadataIp: # Override this via parameter_defaults
description: The IP address of the EC2 metadata server.
type: string
ExternalInterfaceDefaultRoute:
default: ''
description: default route for the external network
type: string
resources:
OsNetConfigImpl:
type: OS::Heat::StructuredConfig
properties:
group: os-apply-config
config:
os_net_config:
network_config:
-
type: interface
name: nic1
use_dhcp: false
addresses:
-
ip_netmask:
list_join:
- '/'
- - {get_param: ControlPlaneIp}
- {get_param: ControlPlaneSubnetCidr}
routes:
-
ip_netmask: 169.254.169.254/32
next_hop: {get_param: EC2MetadataIp}
-
default: true
next_hop: {get_param: ControlPlaneDefaultRoute}
-
type: linux_bond
name: bond_api
bonding_options: "mode=active-backup"
use_dhcp: false
dns_servers: {get_param: DnsServers}
members:
-
type: interface
name: nic2
primary: true
-
type: interface
name: nic3
-
type: vlan
vlan_id: {get_param: InternalApiNetworkVlanID}
device: bond_api
addresses:
-
ip_netmask: {get_param: InternalApiIpSubnet}
-
type: vlan
vlan_id: {get_param: StorageNetworkVlanID}
device: bond_api
addresses:
-
ip_netmask: {get_param: StorageIpSubnet}
-
type: ovs_user_bridge
name: br-link
use_dhcp: false
ovs_extra:
-
str_replace:
template: set port br-link tag=_VLAN_TAG_
params:
_VLAN_TAG_: {get_param: TenantNetworkVlanID}
addresses:
-
ip_netmask: {get_param: TenantIpSubnet}
members:
-
type: ovs_dpdk_port
name: dpdk0
members:
-
type: interface
name: nic4
outputs:
OS::stack_id:
description: The OsNetConfigImpl resource.
value: {get_resource: OsNetConfigImpl}B.2.6. overcloud_deploy.sh
#!/bin/bash openstack overcloud deploy \ --templates \ -e /usr/share/openstack-tripleo-heat-templates/environments/network-isolation.yaml \ -e /usr/share/openstack-tripleo-heat-templates/environments/neutron-ovs-dpdk.yaml \ -e /usr/share/openstack-tripleo-heat-templates/environments/ovs-dpdk-permissions.yaml \ -e /home/stack/ospd-10-vxlan-dpdk-single-port-ctlplane-bonding/network-environment.yaml \ --log-file overcloud_install.log &> overcloud_install.log