The following awscli commands are provided to expose the complexity behind automation tools such Ansible.

Review the following environment variables now and ensure values fit requirements. When values are satisfactory execute the commands in terminal.

$ export clusterid="rhocp"
$ export dns_domain="example.com"
$ export region="us-east-1"
$ export cidrvpc="172.16.0.0/16"
$ export cidrsubnets_public=("172.16.0.0/24" "172.16.1.0/24" "172.16.2.0/24")
$ export cidrsubnets_private=("172.16.16.0/20" "172.16.32.0/20" "172.16.48.0/20")
$ export ec2_type_bastion="t2.medium"
$ export ec2_type_master="m5.2xlarge"
$ export ec2_type_infra="m5.2xlarge"
$ export ec2_type_node="m5.2xlarge"
$ export rhel_release="rhel-7.5"

Create a public private ssh keypair to be used with ssh-agent and ssh authentication on AWS EC2s.

$ if [ ! -f ${HOME}/.ssh/${clusterid}.${dns_domain} ]; then
  echo 'Enter ssh key password'
  read -r passphrase
  ssh-keygen -P ${passphrase} -o -t rsa -f ~/.ssh/${clusterid}.${dns_domain}
fi

$ export sshkey=($(cat ~/.ssh/${clusterid}.${dns_domain}.pub))

Gather available Availability Zones. The first 3 are used.

$ IFS=$' '; export az=($(aws ec2 describe-availability-zones \
    --filters "Name=region-name,Values=${region}" | \
    jq -r '.[][].ZoneName' | \
    head -3 | \
    tr '\n' ' ' | \
    sed -e "s/ $//g"));

$ unset IFS

Retrive Red Hat Cloud Access AMI

$ export ec2ami=($(aws ec2 describe-images \
    --region ${region} --owners 309956199498 | \
    jq -r '.Images[] | [.Name,.ImageId] | @csv' | \
    sed -e 's/,/ /g' | \
    sed -e 's/"//g' | \
    grep HVM_GA | \
    grep Access2-GP2 | \
    grep -i ${rhel_release} | \
    sort | \
    tail -1))

Deploy IAM users and sleep for 15 sec so that AWS can instantiate the users

$ export iamuser=$(aws iam create-user --user-name ${clusterid}.${dns_domain}-admin)

$ export s3user=$(aws iam create-user --user-name ${clusterid}.${dns_domain}-registry)

$ sleep 15

Create access key for IAM users

$ export iamuser_accesskey=$(aws iam create-access-key --user-name ${clusterid}.${dns_domain}-admin)

$ export s3user_accesskey=$(aws iam create-access-key --user-name ${clusterid}.${dns_domain}-registry)

Create and attach policies to IAM users

$ cat << EOF > ~/.iamuser_policy_cpk
{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Action": [
                "ec2:DescribeVolume*",
                "ec2:CreateVolume",
                "ec2:CreateTags",
                "ec2:DescribeInstances",
                "ec2:AttachVolume",
                "ec2:DetachVolume",
                "ec2:DeleteVolume",
                "ec2:DescribeSubnets",
                "ec2:CreateSecurityGroup",
                "ec2:DescribeSecurityGroups",
                "ec2:DescribeRouteTables",
                "ec2:AuthorizeSecurityGroupIngress",
                "ec2:RevokeSecurityGroupIngress",
                "elasticloadbalancing:DescribeTags",
                "elasticloadbalancing:CreateLoadBalancerListeners",
                "elasticloadbalancing:ConfigureHealthCheck",
                "elasticloadbalancing:DeleteLoadBalancerListeners",
                "elasticloadbalancing:RegisterInstancesWithLoadBalancer",
                "elasticloadbalancing:DescribeLoadBalancers",
                "elasticloadbalancing:CreateLoadBalancer",
                "elasticloadbalancing:DeleteLoadBalancer",
                "elasticloadbalancing:ModifyLoadBalancerAttributes",
                "elasticloadbalancing:DescribeLoadBalancerAttributes"
            ],
            "Resource": "*",
            "Effect": "Allow",
            "Sid": "1"
        }
    ]
}
EOF

$ aws iam put-user-policy \
    --user-name ${clusterid}.${dns_domain}-admin \
    --policy-name Admin \
    --policy-document file://~/.iamuser_policy_cpk

$ cat << EOF > ~/.iamuser_policy_s3
{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Action": [
                "s3:*"
            ],
            "Resource": [
                "arn:aws:s3:::${clusterid}.${dns_domain}-registry",
                "arn:aws:s3:::${clusterid}.${dns_domain}-registry/*"
            ],
            "Effect": "Allow",
            "Sid": "1"
        }
    ]
}
EOF

$ aws iam put-user-policy \
    --user-name ${clusterid}.${dns_domain}-registry \
    --policy-name S3 \
    --policy-document file://~/.iamuser_policy_s3

$ rm -rf ~/.iamuser_policy*

Deploy EC2 keypair

$ export keypair=$(aws ec2 import-key-pair \
    --key-name ${clusterid}.${dns_domain} \
    --public-key-material file://~/.ssh/${clusterid}.${dns_domain}.pub \
    )

Deploy S3 bucket and policy

WARN: If region is a region other than us-east-1 then aws s3api create-bucket will require --create-bucket-configuration LocationConstraint=${region}

$ export aws_s3bucket=$(aws s3api create-bucket \
    --bucket $(echo ${clusterid}.${dns_domain}-registry) \
    --region ${region} \
    )

$ aws s3api put-bucket-tagging \
    --bucket $(echo ${clusterid}.${dns_domain}-registry) \
    --tagging "TagSet=[{Key=Clusterid,Value=${clusterid}}]"

$ aws s3api put-bucket-policy \
    --bucket $(echo ${clusterid}.${dns_domain}-registry) \
    --policy "\
{ \
  \"Version\": \"2012-10-17\",
  \"Statement\": [ \
    { \
      \"Action\": \"s3:*\", \
      \"Effect\": \"Allow\", \
      \"Principal\": { \
        \"AWS\": \"$(echo ${s3user} | jq -r '.User.Arn')\" \
      }, \
      \"Resource\": \"arn:aws:s3:::$(echo ${aws_s3bucket} | jq -r '.Location' | sed -e 's/^\///g')\", \
      \"Sid\": \"1\" \
    } \
  ] \
}"

Deploy VPC and DHCP server

$ export vpc=$(aws ec2 create-vpc --cidr-block ${cidrvpc} | jq -r '.')

$ if [ $region == "us-east-1" ]; then
    export vpcdhcpopts_dnsdomain="ec2.internal"
else
    export vpcdhcpopts_dnsdomain="${region}.compute.internal"
fi

$ export vpcdhcpopts=$(aws ec2 create-dhcp-options \
    --dhcp-configuration " \
    [ \
      { \"Key\": \"domain-name\", \"Values\": [ \"${vpcdhcpopts_dnsdomain}\" ] }, \
      { \"Key\": \"domain-name-servers\", \"Values\": [ \"AmazonProvidedDNS\" ] } \
    ]")

$ aws ec2 modify-vpc-attribute \
    --enable-dns-hostnames \
    --vpc-id $(echo ${vpc} | jq -r '.Vpc.VpcId')

$ aws ec2 modify-vpc-attribute \
    --enable-dns-support \
    --vpc-id $(echo ${vpc} | jq -r '.Vpc.VpcId')

$ aws ec2 associate-dhcp-options \
    --dhcp-options-id $(echo ${vpcdhcpopts} | jq -r '.DhcpOptions.DhcpOptionsId') \
    --vpc-id $(echo ${vpc} | jq -r '.Vpc.VpcId')

Deploy IGW and attach to VPC

$ export igw=$(aws ec2 create-internet-gateway)

$ aws ec2 attach-internet-gateway \
  --internet-gateway-id $(echo ${igw} | jq -r '.InternetGateway.InternetGatewayId') \
  --vpc-id $(echo ${vpc} | jq -r '.Vpc.VpcId')

Deploy subnets

$ for i in $(seq 1 3); do
    export subnet${i}_public="$(aws ec2 create-subnet \
        --vpc-id $(echo ${vpc} | jq -r '.Vpc.VpcId') \
        --cidr-block ${cidrsubnets_public[$(expr $i - 1 )]} \
        --availability-zone ${az[$(expr $i - 1)]}
    )"
done

$ for i in $(seq 1 3); do
    export subnet${i}_private="$(aws ec2 create-subnet \
        --vpc-id $(echo ${vpc} | jq -r '.Vpc.VpcId') \
        --cidr-block ${cidrsubnets_private[$(expr $i - 1 )]} \
        --availability-zone ${az[$(expr $i - 1)]}
    )"
done

Deploy EIPs

$ for i in $(seq 0 3); do
    export eip${i}="$(aws ec2 allocate-address --domain vpc)"
done

Deploy NatGW’s

$ for i in $(seq 1 3); do
    j="eip${i}"
    k="subnet${i}_public"
    export natgw${i}="$(aws ec2 create-nat-gateway \
        --subnet-id $(echo ${!k} | jq -r '.Subnet.SubnetId') \
        --allocation-id $(echo ${!j} | jq -r '.AllocationId') \
    )"
done

Deploy RouteTables and routes

$ export routetable0=$(aws ec2 create-route-table \
    --vpc-id $(echo ${vpc} | jq -r '.Vpc.VpcId')
    )

$ aws ec2 create-route \
    --route-table-id $(echo ${routetable0} | jq -r '.RouteTable.RouteTableId') \
    --destination-cidr-block 0.0.0.0/0 \
    --nat-gateway-id $(echo ${igw} | jq -r '.InternetGateway.InternetGatewayId') \
    > /dev/null 2>&1

$ for i in $(seq 1 3); do
    j="subnet${i}_public"
    aws ec2 associate-route-table \
        --route-table-id $(echo ${routetable0} | jq -r '.RouteTable.RouteTableId') \
        --subnet-id $(echo ${!j} | jq -r '.Subnet.SubnetId')
done

$ for i in $(seq 1 3); do
    export routetable${i}="$(aws ec2 create-route-table \
        --vpc-id $(echo ${vpc} | jq -r '.Vpc.VpcId') \
    )"
done

$ for i in $(seq 1 3); do
    j="routetable${i}"
    k="natgw${i}"
    aws ec2 create-route \
        --route-table-id $(echo ${!j} | jq -r '.RouteTable.RouteTableId') \
        --destination-cidr-block 0.0.0.0/0 \
        --nat-gateway-id $(echo ${!k} | jq -r '.NatGateway.NatGatewayId') \
        > /dev/null 2>&1
done

$ for i in $(seq 1 3); do
    j="routetable${i}"
    k="subnet${i}_private"
    aws ec2 associate-route-table \
        --route-table-id $(echo ${!j} | jq -r '.RouteTable.RouteTableId') \
        --subnet-id $(echo ${!k} | jq -r '.Subnet.SubnetId')
done

Deploy SecurityGroups and rules

$ for i in Bastion infra master node; do
    export awssg_$(echo ${i} | tr A-Z a-z)="$(aws ec2 create-security-group \
    --vpc-id $(echo ${vpc} | jq -r '.Vpc.VpcId') \
    --group-name ${i} \
    --description ${i})"
done

$ aws ec2 authorize-security-group-ingress \
    --group-id $(echo ${awssg_bastion} | jq -r '.GroupId') \
    --ip-permissions '[
                      {"IpProtocol": "icmp", "FromPort": 8, "ToPort": -1, "IpRanges": [{"CidrIp": "0.0.0.0/0"}]},
                      {"IpProtocol": "tcp", "FromPort": 22, "ToPort": 22, "IpRanges": [{"CidrIp": "0.0.0.0/0"}]}
                      ]'

$ aws ec2 authorize-security-group-ingress \
    --group-id $(echo ${awssg_infra} | jq -r '.GroupId') \
    --ip-permissions '[
                      {"IpProtocol": "tcp", "FromPort": 80, "ToPort": 80, "IpRanges": [{"CidrIp": "0.0.0.0/0"}]},
                      {"IpProtocol": "tcp", "FromPort": 443, "ToPort": 443, "IpRanges": [{"CidrIp": "0.0.0.0/0"}]},
                      {"IpProtocol": "tcp", "FromPort": 9200, "ToPort": 9200, "IpRanges": [{"CidrIp": "0.0.0.0/0"}]},
                      {"IpProtocol": "tcp", "FromPort": 9300, "ToPort": 9300, "IpRanges": [{"CidrIp": "0.0.0.0/0"}]}
                      ]'

$ aws ec2 authorize-security-group-ingress \
    --group-id $(echo ${awssg_master} | jq -r '.GroupId') \
    --ip-permissions '[
                      {"IpProtocol": "tcp", "FromPort": 443, "ToPort": 443, "IpRanges": [{"CidrIp": "0.0.0.0/0"}]}
                      ]'

$ for i in 2379-2380; do
    aws ec2 authorize-security-group-ingress \
    --group-id $(echo ${awssg_master} | jq -r '.GroupId') \
    --protocol tcp \
    --port $i \
    --source-group $(echo ${awssg_master} | jq -r '.GroupId')
done

$ for i in 2379-2380; do
    aws ec2 authorize-security-group-ingress \
    --group-id $(echo ${awssg_master} | jq -r '.GroupId') \
    --protocol tcp \
    --port $i \
    --source-group $(echo ${awssg_node} | jq -r '.GroupId')
done

$ aws ec2 authorize-security-group-ingress \
    --group-id $(echo ${awssg_node} | jq -r '.GroupId') \
    --ip-permissions '[
                      {"IpProtocol": "icmp", "FromPort": 8, "ToPort": -1, "IpRanges": [{"CidrIp": "0.0.0.0/0"}]}
                      ]'

$ aws ec2 authorize-security-group-ingress \
    --group-id $(echo ${awssg_node} | jq -r '.GroupId') \
    --protocol tcp \
    --port 22 \
    --source-group $(echo ${awssg_bastion} | jq -r '.GroupId')

$ for i in 53 2049 8053 10250; do
    aws ec2 authorize-security-group-ingress \
    --group-id $(echo ${awssg_node} | jq -r '.GroupId') \
    --protocol tcp \
    --port $i \
    --source-group $(echo ${awssg_node} | jq -r '.GroupId')
done

$ for i in 53 4789 8053; do
    aws ec2 authorize-security-group-ingress \
    --group-id $(echo ${awssg_node} | jq -r '.GroupId') \
    --protocol udp \
    --port $i \
    --source-group $(echo ${awssg_node} | jq -r '.GroupId')
done

Deploy ELB’s

$ export elb_masterext=$(aws elb create-load-balancer \
    --load-balancer-name ${clusterid}-public-master \
    --subnets \
      $(echo ${subnet1_public} | jq -r '.Subnet.SubnetId') \
      $(echo ${subnet2_public} | jq -r '.Subnet.SubnetId') \
      $(echo ${subnet3_public} | jq -r '.Subnet.SubnetId') \
    --listener Protocol=TCP,LoadBalancerPort=443,InstanceProtocol=TCP,InstancePort=443 \
    --security-groups $(echo ${awssg_master} | jq -r '.GroupId') \
    --scheme internet-facing \
    --tags Key=name,Value=${clusterid}-public-master Key=Clusterid,Value=${clusterid})

$ aws elb modify-load-balancer-attributes \
    --load-balancer-name ${clusterid}-public-master \
    --load-balancer-attributes "{
        \"CrossZoneLoadBalancing\":{\"Enabled\":true},
        \"ConnectionDraining\":{\"Enabled\":false}
    }"

$ aws elb configure-health-check \
    --load-balancer-name ${clusterid}-public-master \
    --health-check Target=HTTPS:443/api,HealthyThreshold=3,Interval=5,Timeout=2,UnhealthyThreshold=2

$ export elb_masterint=$(aws elb create-load-balancer \
    --load-balancer-name ${clusterid}-private-master \
    --subnets \
      $(echo ${subnet1_private} | jq -r '.Subnet.SubnetId') \
      $(echo ${subnet2_private} | jq -r '.Subnet.SubnetId') \
      $(echo ${subnet3_private} | jq -r '.Subnet.SubnetId') \
    --listener Protocol=TCP,LoadBalancerPort=443,InstanceProtocol=TCP,InstancePort=443 \
    --security-groups $(echo ${awssg_master} | jq -r '.GroupId') \
    --scheme internal \
    --tags Key=name,Value=${clusterid}-private-master Key=Clusterid,Value=${clusterid})

$ aws elb modify-load-balancer-attributes \
    --load-balancer-name ${clusterid}-private-master \
    --load-balancer-attributes "{
        \"CrossZoneLoadBalancing\":{\"Enabled\":true},
        \"ConnectionDraining\":{\"Enabled\":false}
    }"

$ aws elb configure-health-check \
    --load-balancer-name ${clusterid}-private-master \
    --health-check Target=HTTPS:443/api,HealthyThreshold=3,Interval=5,Timeout=2,UnhealthyThreshold=2

$ export elb_infraext=$(aws elb create-load-balancer \
    --load-balancer-name ${clusterid}-public-infra \
    --subnets \
      $(echo ${subnet1_public} | jq -r '.Subnet.SubnetId') \
      $(echo ${subnet2_public} | jq -r '.Subnet.SubnetId') \
      $(echo ${subnet3_public} | jq -r '.Subnet.SubnetId') \
    --listener \
      Protocol=TCP,LoadBalancerPort=80,InstanceProtocol=TCP,InstancePort=80 \
      Protocol=TCP,LoadBalancerPort=443,InstanceProtocol=TCP,InstancePort=443 \
    --security-groups $(echo ${awssg_infra} | jq -r '.GroupId') \
    --scheme internet-facing \
    --tags Key=name,Value=${clusterid}-public-infra Key=Clusterid,Value=${clusterid})

$ aws elb modify-load-balancer-attributes \
    --load-balancer-name ${clusterid}-public-infra \
    --load-balancer-attributes "{
        \"CrossZoneLoadBalancing\":{\"Enabled\":true},
        \"ConnectionDraining\":{\"Enabled\":false}
    }"

$ aws elb configure-health-check \
    --load-balancer-name ${clusterid}-public-infra \
    --health-check Target=TCP:443,HealthyThreshold=2,Interval=5,Timeout=2,UnhealthyThreshold=2

Deploy Route53 zones and resources

$ export route53_extzone=$(aws route53 create-hosted-zone \
  --caller-reference $(date +%s) \
  --name ${clusterid}.${dns_domain} \
  --hosted-zone-config "PrivateZone=False")

$ export route53_intzone=$(aws route53 create-hosted-zone \
  --caller-reference $(date +%s) \
  --name ${clusterid}.${dns_domain} \
  --vpc "VPCRegion=${region},VPCId=$(echo ${vpc} | jq -r '.Vpc.VpcId')" \
  --hosted-zone-config "PrivateZone=True")

$ aws route53 change-resource-record-sets \
    --hosted-zone-id $(echo ${route53_extzone} | jq -r '.HostedZone.Id' | sed 's/\/hostedzone\///g') \
    --change-batch "\
{ \
  \"Changes\": [ \
    { \
      \"Action\": \"CREATE\", \
      \"ResourceRecordSet\": { \
        \"Name\": \"master.${clusterid}.${dns_domain}\", \
        \"Type\": \"CNAME\", \
        \"TTL\": 300, \
        \"ResourceRecords\": [ \
          { \"Value\": \"$(echo ${elb_masterext} | jq -r '.DNSName')\" } \
        ] \
      } \
    } \
  ] \
}"

$ aws route53 change-resource-record-sets \
    --hosted-zone-id $(echo ${route53_intzone} | jq -r '.HostedZone.Id' | sed 's/\/hostedzone\///g') \
    --change-batch "\
{ \
  \"Changes\": [ \
    { \
      \"Action\": \"CREATE\", \
      \"ResourceRecordSet\": { \
        \"Name\": \"master.${clusterid}.${dns_domain}\", \
        \"Type\": \"CNAME\", \
        \"TTL\": 300, \
        \"ResourceRecords\": [ \
          { \"Value\": \"$(echo ${elb_masterint} | jq -r '.DNSName')\" } \
        ] \
      } \
    } \
  ] \
}"

$ aws route53 change-resource-record-sets \
    --hosted-zone-id $(echo ${route53_extzone} | jq -r '.HostedZone.Id' | sed 's/\/hostedzone\///g') \
    --change-batch "\
{ \
  \"Changes\": [ \
    { \
      \"Action\": \"CREATE\", \
      \"ResourceRecordSet\": { \
        \"Name\": \"*.apps.${clusterid}.${dns_domain}\", \
        \"Type\": \"CNAME\", \
        \"TTL\": 300, \
        \"ResourceRecords\": [ \
          { \"Value\": \"$(echo ${elb_infraext} | jq -r '.DNSName')\" } \
        ] \
      } \
    } \
  ] \
}"

$ aws route53 change-resource-record-sets \
    --hosted-zone-id $(echo ${route53_intzone} | jq -r '.HostedZone.Id' | sed 's/\/hostedzone\///g') \
    --change-batch "\
{ \
  \"Changes\": [ \
    { \
      \"Action\": \"CREATE\", \
      \"ResourceRecordSet\": { \
        \"Name\": \"\*.apps.${clusterid}.${dns_domain}\", \
        \"Type\": \"CNAME\", \
        \"TTL\": 300, \
        \"ResourceRecords\": [ \
          { \"Value\": \"$(echo ${elb_infraext} | jq -r '.DNSName')\" } \
        ] \
      } \
    } \
  ] \
}"

Create EC2 user-data script

$ cat << EOF > /tmp/ec2_userdata.sh
#!/bin/bash
if [ "\$#" -ne 2 ]; then exit 2; fi

printf '%s\n' "#cloud-config"

printf '%s' "cloud_config_modules:"
if [ "\$1" == 'bastion' ]; then
  printf '\n%s\n\n' "- package-update-upgrade-install"
else
  printf '\n%s\n\n' "- package-update-upgrade-install
- disk_setup
- mounts
- cc_write_files"
fi

printf '%s' "packages:"
if [ "\$1" == 'bastion' ]; then
  printf '\n%s\n' "- nmap-ncat"
else
  printf '\n%s\n' "- lvm2"
fi

if [ "\$1" != 'bastion' ]; then
  printf '\n%s' 'write_files:
- content: |
    STORAGE_DRIVER=overlay2
    DEVS=/dev/'
  if [[ "\$2" =~ (c5|c5d|i3.metal|m5) ]]; then
    printf '%s' 'nvme1n1'
  else
    printf '%s' 'xvdb'
  fi
  printf '\n%s\n' '    VG=dockervg
    CONTAINER_ROOT_LV_NAME=dockerlv
    CONTAINER_ROOT_LV_MOUNT_PATH=/var/lib/docker
    CONTAINER_ROOT_LV_SIZE=100%FREE
  path: "/etc/sysconfig/docker-storage-setup"
  permissions: "0644"
  owner: "root"'

  printf '\n%s' 'fs_setup:'
  printf '\n%s' '- label: ocp_emptydir
  filesystem: xfs
  device: /dev/'
  if [[ "\$2" =~ (c5|c5d|i3.metal|m5) ]]; then
    printf '%s\n' 'nvme2n1'
  else
    printf '%s\n' 'xvdc'
  fi
  printf '%s' '  partition: auto'
  if [ "\$1" == 'master' ]; then
    printf '\n%s' '- label: etcd
  filesystem: xfs
  device: /dev/'
    if [[ "\$2" =~ (c5|c5d|i3.metal|m5) ]]; then
      printf '%s\n' 'nvme3n1'
    else
      printf '%s\n' 'xvdd'
    fi
    printf '%s' '  partition: auto'
  fi
  printf '\n'

  printf '\n%s' 'mounts:'
  printf '\n%s' '- [ "LABEL=ocp_emptydir", "/var/lib/origin/openshift.local.volumes", xfs, "defaults,gquota" ]'
  if [ "\$1" == 'master' ]; then
    printf '\n%s' '- [ "LABEL=etcd", "/var/lib/etcd", xfs, "defaults,gquota" ]'
  fi
  printf '\n'
fi
EOF

$ chmod u+x /tmp/ec2_userdata.sh

Deploy the bastion EC2, sleep for 15 sec while AWS instantiates the EC2, and associate an EIP with the bastion for direct Internet access.

$ export ec2_bastion=$(aws ec2 run-instances \
    --image-id ${ec2ami[1]} \
    --count 1 \
    --instance-type ${ec2_type_bastion} \
    --key-name ${clusterid}.${dns_domain} \
    --security-group-ids $(echo ${awssg_bastion} | jq -r '.GroupId') \
    --subnet-id $(echo ${subnet1_public} | jq -r '.Subnet.SubnetId') \
    --associate-public-ip-address \
    --block-device-mappings "DeviceName=/dev/sda1,Ebs={DeleteOnTermination=False,VolumeSize=25}" \
    --user-data "$(/tmp/ec2_userdata.sh bastion ${ec2_type_bastion})" \
    --tag-specifications "ResourceType=instance,Tags=[{Key=Name,Value=bastion},{Key=Clusterid,Value=${clusterid}}]" \
  )

$ sleep 15

$ aws ec2 associate-address \
    --allocation-id $(echo ${eip0} | jq -r '.AllocationId') \
    --instance-id $(echo ${ec2_bastion} | jq -r '.Instances[].InstanceId')

Create the master, infra and node EC2s along with EBS volumes

$ for i in $(seq 1 3); do
    j="subnet${i}_private"
    export ec2_master${i}="$(aws ec2 run-instances \
        --image-id ${ec2ami[1]} \
        --count 1 \
        --instance-type ${ec2_type_master} \
        --key-name ${clusterid}.${dns_domain} \
        --security-group-ids $(echo ${awssg_master} | jq -r '.GroupId') $(echo ${awssg_node} | jq -r '.GroupId') \
        --subnet-id $(echo ${!j} | jq -r '.Subnet.SubnetId') \
        --block-device-mappings \
            "DeviceName=/dev/sda1,Ebs={DeleteOnTermination=False,VolumeSize=100}" \
            "DeviceName=/dev/xvdb,Ebs={DeleteOnTermination=False,VolumeSize=100}" \
            "DeviceName=/dev/xvdc,Ebs={DeleteOnTermination=False,VolumeSize=100}" \
            "DeviceName=/dev/xvdd,Ebs={DeleteOnTermination=False,VolumeSize=100}" \
        --user-data "$(/tmp/ec2_userdata.sh master ${ec2_type_master})" \
        --tag-specifications "ResourceType=instance,Tags=[ \
            {Key=Name,Value=master${i}}, \
            {Key=Clusterid,Value=${clusterid}}, \
            {Key=ami,Value=${ec2ami}}, \
            {Key=kubernetes.io/cluster/${clusterid},Value=${clusterid}}]"
        )"
done

$ for i in $(seq 1 3); do
    j="subnet${i}_private"
    export ec2_infra${i}="$(aws ec2 run-instances \
        --image-id ${ec2ami[1]} \
        --count 1 \
        --instance-type ${ec2_type_infra} \
        --key-name ${clusterid}.${dns_domain} \
        --security-group-ids $(echo ${awssg_infra} | jq -r '.GroupId') $(echo ${awssg_node} | jq -r '.GroupId') \
        --subnet-id $(echo ${!j} | jq -r '.Subnet.SubnetId') \
        --block-device-mappings \
            "DeviceName=/dev/sda1,Ebs={DeleteOnTermination=False,VolumeSize=100}" \
            "DeviceName=/dev/xvdb,Ebs={DeleteOnTermination=False,VolumeSize=100}" \
            "DeviceName=/dev/xvdc,Ebs={DeleteOnTermination=False,VolumeSize=100}" \
            "DeviceName=/dev/xvdd,Ebs={DeleteOnTermination=False,VolumeSize=100}" \
        --user-data "$(/tmp/ec2_userdata.sh infra ${ec2_type_infra})" \
        --tag-specifications "ResourceType=instance,Tags=[ \
            {Key=Name,Value=infra${i}}, \
            {Key=Clusterid,Value=${clusterid}}, \
            {Key=ami,Value=${ec2ami}}, \
            {Key=kubernetes.io/cluster/${clusterid},Value=${clusterid}}]"
        )"
done

$ for i in $(seq 1 3); do
    j="subnet${i}_private"
    export ec2_node${i}="$(aws ec2 run-instances \
        --image-id ${ec2ami[1]} \
        --count 1 \
        --instance-type ${ec2_type_node} \
        --key-name ${clusterid}.${dns_domain} \
        --security-group-ids $(echo ${awssg_node} | jq -r '.GroupId') \
        --subnet-id $(echo ${!j} | jq -r '.Subnet.SubnetId') \
        --block-device-mappings \
            "DeviceName=/dev/sda1,Ebs={DeleteOnTermination=False,VolumeSize=100}" \
            "DeviceName=/dev/xvdb,Ebs={DeleteOnTermination=False,VolumeSize=100}" \
            "DeviceName=/dev/xvdc,Ebs={DeleteOnTermination=False,VolumeSize=100}" \
        --user-data "$(/tmp/ec2_userdata.sh node ${ec2_type_node})" \
        --tag-specifications "ResourceType=instance,Tags=[ \
            {Key=Name,Value=node${i}}, \
            {Key=Clusterid,Value=${clusterid}}, \
            {Key=ami,Value=${ec2ami}}, \
            {Key=kubernetes.io/cluster/${clusterid},Value=${clusterid}}]"
        )"
done

$ rm -rf /tmp/ec2_userdata*

Register EC2s to ELB’s

$ export elb_masterextreg=$(aws elb register-instances-with-load-balancer \
    --load-balancer-name ${clusterid}-public-master \
    --instances \
      $(echo ${ec2_master1} | jq -r '.Instances[].InstanceId') \
      $(echo ${ec2_master2} | jq -r '.Instances[].InstanceId') \
      $(echo ${ec2_master3} | jq -r '.Instances[].InstanceId') \
  )

$ export elb_masterintreg=$(aws elb register-instances-with-load-balancer \
    --load-balancer-name ${clusterid}-private-master \
    --instances \
      $(echo ${ec2_master1} | jq -r '.Instances[].InstanceId') \
      $(echo ${ec2_master2} | jq -r '.Instances[].InstanceId') \
      $(echo ${ec2_master3} | jq -r '.Instances[].InstanceId') \
  )

$ export elb_infrareg=$(aws elb register-instances-with-load-balancer \
    --load-balancer-name ${clusterid}-public-infra \
    --instances \
      $(echo ${ec2_infra1} | jq -r '.Instances[].InstanceId') \
      $(echo ${ec2_infra2} | jq -r '.Instances[].InstanceId') \
      $(echo ${ec2_infra3} | jq -r '.Instances[].InstanceId') \
  )

Create tags on AWS components

 aws ec2 create-tags --resources $(echo $vpc | jq -r ".Vpc.VpcId") --tags Key=Name,Value=${clusterid}; \
 aws ec2 create-tags --resources $(echo ${eip0} | jq -r '.AllocationId') --tags Key=Name,Value=bastion; \
 aws ec2 create-tags --resources $(echo ${eip1} | jq -r '.AllocationId') --tags Key=Name,Value=${az[0]}; \
 aws ec2 create-tags --resources $(echo ${eip2} | jq -r '.AllocationId') --tags Key=Name,Value=${az[1]}; \
 aws ec2 create-tags --resources $(echo ${eip3} | jq -r '.AllocationId') --tags Key=Name,Value=${az[2]}; \
 aws ec2 create-tags --resources $(echo ${natgw1} | jq -r '.NatGateway.NatGatewayId') --tags Key=Name,Value=${az[0]}; \
 aws ec2 create-tags --resources $(echo ${natgw2} | jq -r '.NatGateway.NatGatewayId') --tags Key=Name,Value=${az[1]}; \
 aws ec2 create-tags --resources $(echo ${natgw3} | jq -r '.NatGateway.NatGatewayId') --tags Key=Name,Value=${az[2]}; \
 aws ec2 create-tags --resources $(echo ${routetable0} | jq -r '.RouteTable.RouteTableId') --tags Key=Name,Value=routing; \
 aws ec2 create-tags --resources $(echo ${routetable1} | jq -r '.RouteTable.RouteTableId') --tags Key=Name,Value=${az[0]}; \
 aws ec2 create-tags --resources $(echo ${routetable2} | jq -r '.RouteTable.RouteTableId') --tags Key=Name,Value=${az[1]}; \
 aws ec2 create-tags --resources $(echo ${routetable3} | jq -r '.RouteTable.RouteTableId') --tags Key=Name,Value=${az[2]}; \
 aws ec2 create-tags --resources $(echo ${awssg_bastion} | jq -r '.GroupId') --tags Key=Name,Value=Bastion; \
 aws ec2 create-tags --resources $(echo ${awssg_bastion} | jq -r '.GroupId') --tags Key=clusterid,Value=${clusterid}; \
 aws ec2 create-tags --resources $(echo ${awssg_master} | jq -r '.GroupId') --tags Key=Name,Value=Master; \
 aws ec2 create-tags --resources $(echo ${awssg_master} | jq -r '.GroupId') --tags Key=clusterid,Value=${clusterid}; \
 aws ec2 create-tags --resources $(echo ${awssg_infra} | jq -r '.GroupId') --tags Key=Name,Value=Infra; \
 aws ec2 create-tags --resources $(echo ${awssg_infra} | jq -r '.GroupId') --tags Key=clusterid,Value=${clusterid}; \
 aws ec2 create-tags --resources $(echo ${awssg_node} | jq -r '.GroupId') --tags Key=Name,Value=Node; \
 aws ec2 create-tags --resources $(echo ${awssg_node} | jq -r '.GroupId') --tags Key=clusterid,Value=${clusterid}

Create configuration files. These files are used to assist with installing

$ cat << EOF > ~/.ssh/config-${clusterid}.${dns_domain}
#<!-- BEGIN OUTPUT -->
Host bastion
  HostName                 $(echo ${eip0} | jq -r '.PublicIp')
  User                     ec2-user
  StrictHostKeyChecking    no
  ProxyCommand             none
  CheckHostIP              no
  ForwardAgent             yes
  ServerAliveInterval      15
  TCPKeepAlive             yes
  ControlMaster            auto
  ControlPath              ~/.ssh/mux-%r@%h:%p
  ControlPersist           15m
  ServerAliveInterval      30
  IdentityFile             ~/.ssh/${clusterid}.${dns_domain}

Host *.compute-1.amazonaws.com
  ProxyCommand             ssh -w 300 -W %h:%p bastion
  user                     ec2-user
  StrictHostKeyChecking    no
  CheckHostIP              no
  ServerAliveInterval      30
  IdentityFile             ~/.ssh/${clusterid}.${dns_domain}

Host *.ec2.internal
  ProxyCommand             ssh -w 300 -W %h:%p bastion
  user                     ec2-user
  StrictHostKeyChecking    no
  CheckHostIP              no
  ServerAliveInterval      30
  IdentityFile             ~/.ssh/${clusterid}.${dns_domain}
#<!-- END OUTPUT -->
EOF

$ cat << EOF > ~/.ssh/config-${clusterid}.${dns_domain}-domaindelegation
#<!-- BEGIN OUTPUT -->
$(echo ${route53_extzone} | jq -r '.DelegationSet.NameServers[]')
#<!-- END OUTPUT -->
EOF

$ cat << EOF > ~/.ssh/config-${clusterid}.${dns_domain}-cpkuser_access_key
#<!-- BEGIN OUTPUT -->
openshift_cloudprovider_aws_access_key=$(echo ${iamuser_accesskey} | jq -r '.AccessKey.AccessKeyId')
openshift_cloudprovider_aws_secret_key=$(echo ${iamuser_accesskey} | jq -r '.AccessKey.SecretAccessKey')
#<!-- END OUTPUT -->
EOF

$ cat << EOF > ~/.ssh/config-${clusterid}.${dns_domain}-cpk
#<!-- BEGIN OUTPUT -->
openshift_cloudprovider_kind=aws
openshift_clusterid=${clusterid}
EOF
cat ~/.ssh/config-${clusterid}.${dns_domain}-cpkuser_access_key | \
    grep -v 'OUTPUT -->' >> \
    ~/.ssh/config-${clusterid}.${dns_domain}-cpk
cat << EOF >> ~/.ssh/config-${clusterid}.${dns_domain}-cpk
#<!-- END OUTPUT -->
EOF

$ cat << EOF > ~/.ssh/config-${clusterid}.${dns_domain}-s3user_access_key
#<!-- BEGIN OUTPUT -->
openshift_hosted_registry_storage_s3_accesskey=$(echo ${s3user_accesskey} | jq -r '.AccessKey.AccessKeyId')
openshift_hosted_registry_storage_s3_secretkey=$(echo ${s3user_accesskey} | jq -r '.AccessKey.SecretAccessKey')
#<!-- END OUTPUT -->
EOF

$ cat << EOF > ~/.ssh/config-${clusterid}.${dns_domain}-s3
#<!-- BEGIN OUTPUT -->
openshift_hosted_manage_registry=true
openshift_hosted_registry_storage_kind=object
openshift_hosted_registry_storage_provider=s3
EOF
cat ~/.ssh/config-${clusterid}.${dns_domain}-s3user_access_key | \
    grep -v 'OUTPUT -->' >> \
    ~/.ssh/config-${clusterid}.${dns_domain}-s3
cat << EOF >> ~/.ssh/config-${clusterid}.${dns_domain}-s3
openshift_hosted_registry_storage_s3_bucket=${clusterid}.${dns_domain}-registry
openshift_hosted_registry_storage_s3_region=${region}
openshift_hosted_registry_storage_s3_chunksize=26214400
openshift_hosted_registry_storage_s3_rootdirectory=/registry
openshift_hosted_registry_pullthrough=true
openshift_hosted_registry_acceptschema2=true
openshift_hosted_registry_enforcequota=true
openshift_hosted_registry_replicas=3
openshift_hosted_registry_selector='region=infra'
#<!-- END OUTPUT -->
EOF

$ cat << EOF > ~/.ssh/config-${clusterid}.${dns_domain}-urls
#<!-- BEGIN OUTPUT -->
openshift_master_default_subdomain=apps.${clusterid}.${dns_domain}
openshift_master_cluster_hostname=master.${clusterid}.${dns_domain}
openshift_master_cluster_public_hostname=master.${clusterid}.${dns_domain}
#<!-- END OUTPUT -->
EOF

$ cat << EOF > ~/.ssh/config-${clusterid}.${dns_domain}-hosts
[masters]
$(echo ${ec2_master1} | jq -r '.Instances[].PrivateDnsName') openshift_node_labels="{'region': 'master'}"
$(echo ${ec2_master2} | jq -r '.Instances[].PrivateDnsName') openshift_node_labels="{'region': 'master'}"
$(echo ${ec2_master3} | jq -r '.Instances[].PrivateDnsName') openshift_node_labels="{'region': 'master'}"

[etcd]

[etcd:children]
masters

[nodes]
$(echo ${ec2_node1} | jq -r '.Instances[].PrivateDnsName') openshift_node_labels="{'region': 'apps'}"
$(echo ${ec2_node2} | jq -r '.Instances[].PrivateDnsName') openshift_node_labels="{'region': 'apps'}"
$(echo ${ec2_node3} | jq -r '.Instances[].PrivateDnsName') openshift_node_labels="{'region': 'apps'}"
$(echo ${ec2_infra1} | jq -r '.Instances[].PrivateDnsName') openshift_node_labels="{'region': 'infra', 'zone': 'default'}"
$(echo ${ec2_infra2} | jq -r '.Instances[].PrivateDnsName') openshift_node_labels="{'region': 'infra', 'zone': 'default'}"
$(echo ${ec2_infra3} | jq -r '.Instances[].PrivateDnsName') openshift_node_labels="{'region': 'infra', 'zone': 'default'}"

[nodes:children]
masters
EOF

GitHub OpenShift openshift-ansible-contrib repository contains an Ansible playbook that provides a friendly and repeatable deployment experience.

Clone openshift-ansible-contrib repository then enter the reference architecture 3.9 directory.

$ git clone https://github.com/openshift/openshift-ansible-contrib.git

$ cd openshift-ansible-contrib/reference-architecture/3.9

Create a simple Ansible inventory

$ sudo vi /etc/ansible/hosts
[local]
127.0.0.1

[local:vars]
ansible_connection=local
ansible_become=False

Review playbooks/vars/main.yaml now and ensure values fit requirements.

When values are satisfactory execute deploy_aws.yaml play to deploy AWS infrastructure.

$ ansible-playbook playbooks/deploy_aws.yaml

Set environment variables to be sourced by other commands. These values can be modified to fit any environment.

$ export ec2_type_cns="m5.2xlarge"

Deploy SecurityGroup and rules

$ export awssg_cns=$(aws ec2 create-security-group \
    --vpc-id $(echo ${vpc} | jq -r '.Vpc.VpcId') \
    --group-name cns \
    --description "cns")

$ for i in 111 2222 3260 24007-24008 24010 49152-49664; do
    aws ec2 authorize-security-group-ingress \
    --group-id $(echo ${awssg_cns} | jq -r '.GroupId') \
    --protocol tcp \
    --port $i \
    --source-group $(echo ${awssg_cns} | jq -r '.GroupId')
done

$ for i in 3260 24007-24008 24010 49152-49664; do
    aws ec2 authorize-security-group-ingress \
    --group-id $(echo ${awssg_cns} | jq -r '.GroupId') \
    --protocol tcp \
    --port $i \
    --source-group $(echo ${awssg_node} | jq -r '.GroupId')
done

$ aws ec2 authorize-security-group-ingress \
    --group-id $(echo ${awssg_cns} | jq -r '.GroupId') \
    --protocol udp \
    --port 111 \
    --source-group $(echo ${awssg_cns} | jq -r '.GroupId')

Create node EC2 instances along with EBS volumes

$ for i in 1 2 3; do
    j="subnet${i}_private"
    export ec2_cns${i}="$(aws ec2 run-instances \
        --image-id ${ec2ami[1]} \
        --count 1 \
        --instance-type ${ec2_type_cns} \
        --key-name ${clusterid}.${dns_domain} \
        --security-group-ids $(echo ${awssg_cns} | jq -r '.GroupId') $(echo ${awssg_node} | jq -r '.GroupId') \
        --subnet-id $(echo ${!j} | jq -r '.Subnet.SubnetId') \
        --block-device-mappings \
            "DeviceName=/dev/sda1,Ebs={DeleteOnTermination=False,VolumeSize=100}" \
            "DeviceName=/dev/xvdb,Ebs={DeleteOnTermination=False,VolumeSize=100}" \
            "DeviceName=/dev/xvdc,Ebs={DeleteOnTermination=False,VolumeSize=100}" \
            "DeviceName=/dev/xvdd,Ebs={DeleteOnTermination=False,VolumeSize=100}" \
        --user-data "$(/tmp/ec2_userdata.sh node ${ec2_type_node})" \
        --tag-specifications "ResourceType=instance,Tags=[ \
            {Key=Name,Value=cns${i}}, \
            {Key=Clusterid,Value=${clusterid}}, \
            {Key=ami,Value=${ec2ami}}, \
            {Key=kubernetes.io/cluster/${clusterid},Value=${clusterid}}]"
        )"
done

Create configuration files. These files are used to assist with installing Red Hat OpenShift Container Platform.

$ cat << EOF > ~/.ssh/config-${clusterid}.${dns_domain}-hostscns
$(echo ${ec2_cns1} | jq -r '.Instances[].PrivateDnsName') openshift_schedulable=True
$(echo ${ec2_cns2} | jq -r '.Instances[].PrivateDnsName') openshift_schedulable=True
$(echo ${ec2_cns3} | jq -r '.Instances[].PrivateDnsName') openshift_schedulable=True

EOF

$ cat << EOF > ~/.ssh/config-${clusterid}.${dns_domain}-hostsgfs
[glusterfs]
$(echo ${ec2_cns1} | jq -r '.Instances[].PrivateDnsName') glusterfs_devices='[ "/dev/nvme3n1" ]'
$(echo ${ec2_cns2} | jq -r '.Instances[].PrivateDnsName') glusterfs_devices='[ "/dev/nvme3n1" ]'
$(echo ${ec2_cns3} | jq -r '.Instances[].PrivateDnsName') glusterfs_devices='[ "/dev/nvme3n1" ]'
EOF

Run deploy_aws_cns.yaml play to deploy additional infrastructure for CNS.

$ ansible-playbook playbooks/deploy_aws_cns.yaml

To access Red Hat OpenShift Container Platform on AWS from the public Internet delegation for the subdomain must be setup using the following information.

WARN: DNS service provder domain delegation is out of scope of this guide. A customer will need to follow their providers best practise in delegation setup.

Red Hat OpenShift Container Platform provides the ability to use many different authentication platforms.

Detailed listing of other authentication providers are available at Configuring Authentication and User Agent.

For this reference architecture Google’s OpenID Connect Integration is used. When configuring the authentication, the following parameters must be added to the ansible inventory. An example is shown below.

openshift_master_identity_providers=[{'name': 'google', 'challenge': 'false', 'login': 'true', 'kind': 'GoogleIdentityProvider', 'mapping_method': 'claim', 'clientID': '246358064255-5ic2e4b1b9ipfa7hddfkhuf8s6eq2rfj.apps.googleusercontent.com', 'clientSecret': 'Za3PWZg7gQxM26HBljgBMBBF', 'hostedDomain': 'redhat.com'}]

This section provides an example inventory file required for an advanced installation of Red Hat OpenShift Container Platform.

The inventory file contains both variables and instances used for the configuration and deployment of Red Hat OpenShift Container Platform.

$ sudo vi /etc/ansible/hosts
[OSEv3:children]
masters
etcd
nodes

[OSEv3:vars]
debug_level=2
ansible_user=ec2-user
ansible_become=yes
openshift_deployment_type=openshift-enterprise
openshift_release=v3.9
openshift_master_api_port=443
openshift_master_console_port=443
openshift_portal_net=172.30.0.0/16
os_sdn_network_plugin_name='redhat/openshift-ovs-networkpolicy'
openshift_master_cluster_method=native
container_runtime_docker_storage_setup_device=/dev/nvme1n1
openshift_node_local_quota_per_fsgroup=512Mi
osm_use_cockpit=true
openshift_hostname_check=false
openshift_examples_modify_imagestreams=true
oreg_url=registry.access.redhat.com/openshift3/ose-${component}:${version}

openshift_hosted_router_selector='region=infra'
openshift_hosted_router_replicas=3

# UPDATE TO CORRECT IDENTITY PROVIDER
openshift_master_identity_providers=[{'name': 'google', 'challenge': 'false', 'login': 'true', 'kind': 'GoogleIdentityProvider', 'mapping_method': 'claim', 'clientID': '246358064255-5ic2e4b1b9ipfa7hddfkhuf8s6eq2rfj.apps.googleusercontent.com', 'clientSecret': 'Za3PWZg7gQxM26HBljgBMBBF', 'hostedDomain': 'redhat.com'}]

# UPDATE USING VALUES FOUND IN awscli env vars or ~/playbooks/var/main.yaml
# SEE ALSO FILE ~/.ssh/config-< CLUSTERID >.< DNS_DOMAIN >-urls
openshift_master_default_subdomain=apps.< CLUSTERID >.< DNS_DOMAIN >
openshift_master_cluster_hostname=master.< CLUSTERID >.< DNS_DOMAIN >
openshift_master_cluster_public_hostname=master.< CLUSTERID >.< DNS_DOMAIN >

# UPDATE USING VALUES FOUND IN FILE ~/.ssh/config-< CLUSTERID >.< DNS_DOMAIN >-cpk
openshift_cloudprovider_kind=aws
openshift_clusterid=refarch
openshift_cloudprovider_aws_access_key=UPDATEACCESSKEY
openshift_cloudprovider_aws_secret_key=UPDATESECRETKEY

# UPDATE USING VALUES FOUND IN FILE ~/.ssh/config-< CLUSTERID >.< DNS_DOMAIN >-s3
openshift_hosted_manage_registry=true
openshift_hosted_registry_storage_kind=object
openshift_hosted_registry_storage_provider=s3
openshift_hosted_registry_storage_s3_accesskey=UPDATEACCESSKEY
openshift_hosted_registry_storage_s3_secretkey=UPDATESECRETKEY
openshift_hosted_registry_storage_s3_bucket=refarch-registry
openshift_hosted_registry_storage_s3_region=REGION
openshift_hosted_registry_storage_s3_chunksize=26214400
openshift_hosted_registry_storage_s3_rootdirectory=/registry
openshift_hosted_registry_pullthrough=true
openshift_hosted_registry_acceptschema2=true
openshift_hosted_registry_enforcequota=true
openshift_hosted_registry_replicas=3
openshift_hosted_registry_selector='region=infra'

# Aggregated logging
openshift_logging_install_logging=true
openshift_logging_storage_kind=dynamic
openshift_logging_storage_volume_size=25Gi
openshift_logging_es_cluster_size=3

# Metrics
openshift_metrics_install_metrics=true
openshift_metrics_storage_kind=dynamic
openshift_metrics_storage_volume_size=25Gi

openshift_enable_service_catalog=true

template_service_broker_install=true

# UPDATE USING HOSTS FOUND IN FILE ~/.ssh/config-< CLUSTERID >.< DNS_DOMAIN >-hosts
[masters]
ip-172-16-30-15.ec2.internal openshift_node_labels="{'region': 'master'}"
ip-172-16-47-176.ec2.internal openshift_node_labels="{'region': 'master'}"
ip-172-16-48-251.ec2.internal openshift_node_labels="{'region': 'master'}"

[etcd]

[etcd:children]
masters

# UPDATE USING HOSTS FOUND IN FILE ~/.ssh/config-< CLUSTERID >.< DNS_DOMAIN >-hosts
[nodes]
ip-172-16-16-239.ec2.internal openshift_node_labels="{'region': 'apps'}"
ip-172-16-37-179.ec2.internal openshift_node_labels="{'region': 'apps'}"
ip-172-16-60-134.ec2.internal openshift_node_labels="{'region': 'apps'}"
ip-172-16-17-209.ec2.internal openshift_node_labels="{'region': 'infra', 'zone': 'default'}"
ip-172-16-46-136.ec2.internal openshift_node_labels="{'region': 'infra', 'zone': 'default'}"
ip-172-16-56-149.ec2.internal openshift_node_labels="{'region': 'infra', 'zone': 'default'}"

[nodes:children]
masters

If CNS is used in the OpenShift installation specific variables must be set in the inventory.

$ sudo vi /etc/ansible/hosts
# ADD glusterfs to section [OSEv3:children]
[OSEv3:children]
masters
etcd
nodes
glusterfs

[OSEv3:vars]

....omitted...

# CNS storage cluster
openshift_storage_glusterfs_namespace=glusterfs
openshift_storage_glusterfs_storageclass=true
openshift_storage_glusterfs_storageclass_default=false
openshift_storage_glusterfs_block_deploy=false
openshift_storage_glusterfs_block_host_vol_create=false
openshift_storage_glusterfs_block_host_vol_size=80
openshift_storage_glusterfs_block_storageclass=false
openshift_storage_glusterfs_block_storageclass_default=false

....omitted...

# ADD CNS HOSTS TO SECTION [nodes]
# SEE INVENTORY IN FILE ~/.ssh/config-< CLUSTERID >.< DNS_DOMAIN >-hostscns
[nodes]
....omitted...
ip-172-16-17-130.ec2.internal openshift_schedulable=True
ip-172-16-40-219.ec2.internal openshift_schedulable=True
ip-172-16-63-212.ec2.internal openshift_schedulable=True

# ADD SECTION [glusterfs]
# SEE INVENTORY IN FILE ~/.ssh/config-< CLUSTERID >.< DNS_DOMAIN >-hostsgfs
[glusterfs]
ip-172-16-17-130.ec2.internal glusterfs_devices='[ "/dev/nvme3n1" ]'
ip-172-16-40-219.ec2.internal glusterfs_devices='[ "/dev/nvme3n1" ]'
ip-172-16-63-212.ec2.internal glusterfs_devices='[ "/dev/nvme3n1" ]'

During the deployment of instances an extra volume is added to the instances for EmptyDir storage. EmptyDir provides ephemeral (as opposed to persistent) storage for containers. The volume is used to help ensure that the /var volume does not get filled by containers using the storage.

The user-data script attached to EC2 instances automatically provisioned filesystem, added an entry to fstab, and mounted this volume.

During the deployment of the master instances an extra volume was added to the instances for etcd storage. Having the separate disks specifically for etcd ensures that all of the resources are available to the etcd service such as i/o and total disk space.

The user-data script attached to EC2 instances automatically provisioned filesystem, added an entry to fstab, and mounted this volume.

The prerequisite playbook provided by the OpenShift Ansible RPMs configures container storage and installs any remaining packages for the installation.

$ ansible-playbook \
    /usr/share/ansible/openshift-ansible/playbooks/prerequisites.yml