nova qemu-kvm memory is never released even though memory in the guest has been freed.

Issue

• nova qemu-kvm memory is never released even though memory in the guest has been freed.

• In this use case, we run k8 inside 8 or 16 workers but over time the memory gets used on all the vms but the memory in the qemu-kvm process is never released back to the hypervisor.

• In the following example, qemu-kvm is using ~4GB of ram:

[root@hypervisor ~]# ps -aufxg | grep undercloud
qemu     1395544  286  1.6 22472264 4283628 ?    Sl   17:27   2:02 /usr/bin/qemu-system-x86_64 -machine accel=kvm -name guest=undercloud-0-rhosp16,debug-threads=on -S -object secret,id=masterKey0,format=raw,file=/var/lib/libvirt/qemu/domain-29-undercloud-0-rhosp16/master-key.aes -machine pc-i440fx-5.0,accel=kvm,usb=off,vmport=off,dump-guest-core=off -cpu IvyBridge-IBRS,vmx=on,pdpe1gb=on -m 16384 -overcommit mem-lock=off -smp 4,sockets=4,cores=1,threads=1 -numa node,nodeid=0,cpus=0-1,mem=8192 -numa node,nodeid=1,cpus=2-3,mem=8192 -uuid dfc2b446-d3b9-4568-9e21-20674c92fd86 -no-user-config -nodefaults -chardev socket,id=charmonitor,fd=36,server,nowait -mon chardev=charmonitor,id=monitor,mode=control -rtc base=utc,driftfix=slew -global kvm-pit.lost_tick_policy=delay -no-hpet -no-shutdown -global PIIX4_PM.disable_s3=1 -global PIIX4_PM.disable_s4=1 -boot menu=on,strict=on -device ich9-usb-ehci1,id=usb,bus=pci.0,addr=0x6.0x7 -device ich9-usb-uhci1,masterbus=usb.0,firstport=0,bus=pci.0,multifunction=on,addr=0x6 -device ich9-usb-uhci2,masterbus=usb.0,firstport=2,bus=pci.0,addr=0x6.0x1 -device ich9-usb-uhci3,masterbus=usb.0,firstport=4,bus=pci.0,addr=0x6.0x2 -device virtio-serial-pci,id=virtio-serial0,bus=pci.0,addr=0x5 -blockdev {"driver":"file","filename":"/var/lib/jenkins/VMs/undercloud-0-rhosp16.qcow2","node-name":"libvirt-5-storage","auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-5-format","read-only":false,"driver":"qcow2","file":"libvirt-5-storage","backing":null} -device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x9,drive=libvirt-5-format,id=virtio-disk0,bootindex=2 -blockdev {"driver":"file","filename":"/var/lib/jenkins/VMs/undercloud-0-rhosp16-vdb.qcow2","node-name":"libvirt-4-storage","auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-4-format","read-only":false,"driver":"qcow2","file":"libvirt-4-storage","backing":null} -device virtio-blk-pci,scsi=off,bus=pci.0,addr=0xa,drive=libvirt-4-format,id=virtio-disk1 -blockdev {"driver":"file","filename":"/var/lib/jenkins/VMs/undercloud-0-rhosp16-vdc.qcow2","node-name":"libvirt-3-storage","auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-3-format","read-only":false,"driver":"qcow2","file":"libvirt-3-storage","backing":null} -device virtio-blk-pci,scsi=off,bus=pci.0,addr=0xb,drive=libvirt-3-format,id=virtio-disk2 -blockdev {"driver":"file","filename":"/var/lib/jenkins/VMs/undercloud-0-rhosp16-vdd.qcow2","node-name":"libvirt-2-storage","auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-2-format","read-only":false,"driver":"qcow2","file":"libvirt-2-storage","backing":null} -device virtio-blk-pci,scsi=off,bus=pci.0,addr=0xc,drive=libvirt-2-format,id=virtio-disk3 -device ide-cd,bus=ide.0,unit=0,id=ide0-0-0,bootindex=3 -netdev tap,fds=39:40:56:61,id=hostnet0,vhost=on,vhostfds=62:63:64:65 -device virtio-net-pci,mq=on,vectors=10,netdev=hostnet0,id=net0,mac=52:54:00:55:d8:7a,bus=pci.0,addr=0x3 -netdev tap,fds=66:67:68:69,id=hostnet1,vhost=on,vhostfds=70:71:72:73 -device virtio-net-pci,mq=on,vectors=10,netdev=hostnet1,id=net1,mac=52:54:00:e3:fd:03,bus=pci.0,addr=0x8 -chardev pty,id=charserial0 -device isa-serial,chardev=charserial0,id=serial0 -chardev spicevmc,id=charchannel0,name=vdagent -device virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel0,name=com.redhat.spice.0 -spice port=5900,addr=127.0.0.1,disable-ticketing,image-compression=off,seamless-migration=on -device qxl-vga,id=video0,ram_size=67108864,vram_size=67108864,vram64_size_mb=0,vgamem_mb=16,max_outputs=1,bus=pci.0,addr=0x2 -device intel-hda,id=sound0,bus=pci.0,addr=0x4 -device hda-duplex,id=sound0-codec0,bus=sound0.0,cad=0 -chardev spicevmc,id=charredir0,name=usbredir -device usb-redir,chardev=charredir0,id=redir0,bus=usb.0,port=1 -chardev spicevmc,id=charredir1,name=usbredir -device usb-redir,chardev=charredir1,id=redir1,bus=usb.0,port=2 -device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x7,deflate-on-oom=on -sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny -msg timestamp=on

• We then allocate 12GB of RAM more in the guest which brings the total of allocated memory to 16GB of RAM:

[root@hypervisor ~]# ps -aufxg | grep undercloud
root     1398665  0.0  0.0 221268   728 pts/2    S+   17:30   0:00  |           \_ grep --color=auto undercloud
qemu     1395544  174  6.3 22472264 16727856 ?   Rl   17:27   4:19 /usr/bin/qemu-system-x86_64 -machine accel=kvm -name guest=undercloud-0-rhosp16,debug-threads=on -S -object secret,id=masterKey0,format=raw,file=/var/lib/libvirt/qemu/domain-29-undercloud-0-rhosp16/master-key.aes -machine pc-i440fx-5.0,accel=kvm,usb=off,vmport=off,dump-guest-core=off -cpu IvyBridge-IBRS,vmx=on,pdpe1gb=on -m 16384 -overcommit mem-lock=off -smp 4,sockets=4,cores=1,threads=1 -numa node,nodeid=0,cpus=0-1,mem=8192 -numa node,nodeid=1,cpus=2-3,mem=8192 -uuid dfc2b446-d3b9-4568-9e21-20674c92fd86 -no-user-config -nodefaults -chardev socket,id=charmonitor,fd=36,server,nowait -mon chardev=charmonitor,id=monitor,mode=control -rtc base=utc,driftfix=slew -global kvm-pit.lost_tick_policy=delay -no-hpet -no-shutdown -global PIIX4_PM.disable_s3=1 -global PIIX4_PM.disable_s4=1 -boot menu=on,strict=on -device ich9-usb-ehci1,id=usb,bus=pci.0,addr=0x6.0x7 -device ich9-usb-uhci1,masterbus=usb.0,firstport=0,bus=pci.0,multifunction=on,addr=0x6 -device ich9-usb-uhci2,masterbus=usb.0,firstport=2,bus=pci.0,addr=0x6.0x1 -device ich9-usb-uhci3,masterbus=usb.0,firstport=4,bus=pci.0,addr=0x6.0x2 -device virtio-serial-pci,id=virtio-serial0,bus=pci.0,addr=0x5 -blockdev {"driver":"file","filename":"/var/lib/jenkins/VMs/undercloud-0-rhosp16.qcow2","node-name":"libvirt-5-storage","auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-5-format","read-only":false,"driver":"qcow2","file":"libvirt-5-storage","backing":null} -device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x9,drive=libvirt-5-format,id=virtio-disk0,bootindex=2 -blockdev {"driver":"file","filename":"/var/lib/jenkins/VMs/undercloud-0-rhosp16-vdb.qcow2","node-name":"libvirt-4-storage","auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-4-format","read-only":false,"driver":"qcow2","file":"libvirt-4-storage","backing":null} -device virtio-blk-pci,scsi=off,bus=pci.0,addr=0xa,drive=libvirt-4-format,id=virtio-disk1 -blockdev {"driver":"file","filename":"/var/lib/jenkins/VMs/undercloud-0-rhosp16-vdc.qcow2","node-name":"libvirt-3-storage","auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-3-format","read-only":false,"driver":"qcow2","file":"libvirt-3-storage","backing":null} -device virtio-blk-pci,scsi=off,bus=pci.0,addr=0xb,drive=libvirt-3-format,id=virtio-disk2 -blockdev {"driver":"file","filename":"/var/lib/jenkins/VMs/undercloud-0-rhosp16-vdd.qcow2","node-name":"libvirt-2-storage","auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-2-format","read-only":false,"driver":"qcow2","file":"libvirt-2-storage","backing":null} -device virtio-blk-pci,scsi=off,bus=pci.0,addr=0xc,drive=libvirt-2-format,id=virtio-disk3 -device ide-cd,bus=ide.0,unit=0,id=ide0-0-0,bootindex=3 -netdev tap,fds=39:40:56:61,id=hostnet0,vhost=on,vhostfds=62:63:64:65 -device virtio-net-pci,mq=on,vectors=10,netdev=hostnet0,id=net0,mac=52:54:00:55:d8:7a,bus=pci.0,addr=0x3 -netdev tap,fds=66:67:68:69,id=hostnet1,vhost=on,vhostfds=70:71:72:73 -device virtio-net-pci,mq=on,vectors=10,netdev=hostnet1,id=net1,mac=52:54:00:e3:fd:03,bus=pci.0,addr=0x8 -chardev pty,id=charserial0 -device isa-serial,chardev=charserial0,id=serial0 -chardev spicevmc,id=charchannel0,name=vdagent -device virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel0,name=com.redhat.spice.0 -spice port=5900,addr=127.0.0.1,disable-ticketing,image-compression=off,seamless-migration=on -device qxl-vga,id=video0,ram_size=67108864,vram_size=67108864,vram64_size_mb=0,vgamem_mb=16,max_outputs=1,bus=pci.0,addr=0x2 -device intel-hda,id=sound0,bus=pci.0,addr=0x4 -device hda-duplex,id=sound0-codec0,bus=sound0.0,cad=0 -chardev spicevmc,id=charredir0,name=usbredir -device usb-redir,chardev=charredir0,id=redir0,bus=usb.0,port=1 -chardev spicevmc,id=charredir1,name=usbredir -device usb-redir,chardev=charredir1,id=redir1,bus=usb.0,port=2 -device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x7,deflate-on-oom=on -sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny -msg timestamp=on

• We see it's being used in the guest:

[stack@undercloud-0-rhosp16 memoryallocator]$ free -m
              total        used        free      shared  buff/cache   available
Mem:          15876       15282         222           3         371         218
Swap:          6143        1613        4530

• We stop the process and then validate it's really freed in the guest:

[stack@undercloud-0-rhosp16 memoryallocator]$ free -m
              total        used        free      shared  buff/cache   available
Mem:          15876        3819       11588           5         467       11651
Swap:          6143        1085        5058