Nova fails to delete rbd image with "ImageBusy: error removing image"

Issue

• We encountered a bug and it is the same as this one found in upstream bug:

https://bugs.launchpad.net/nova/+bug/1438331

• When removing guests that have been booted on Ceph, Nova will occasionally put guests in to ERROR state with the following ...

Reported to the controller:

| fault | {"message": "error removing image", "code": 500, "details": " File \"/usr/lib/python2.7/site-packages/nova/compute/manager.py\", line 314, in decorated_function |
| | return function(self, context, *args, **kwargs) |
| | File \"/usr/lib/python2.7/site-packages/nova/compute/manager.py\", line 2525, in terminate_instance |
| | do_terminate_instance(instance, bdms) |
| | File \"/usr/lib/python2.7/site-packages/nova/openstack/common/lockutils.py\", line 272, in inner |
| | return f(*args, **kwargs) |
| | File \"/usr/lib/python2.7/site-packages/nova/compute/manager.py\", line 2523, in do_terminate_instance |
| | self._set_instance_error_state(context, instance) |
| | File \"/usr/lib/python2.7/site-packages/nova/openstack/common/excutils.py\", line 82, in __exit__ |
| | six.reraise(self.type_, self.value, self.tb) |
| | File \"/usr/lib/python2.7/site-packages/nova/compute/manager.py\", line 2513, in do_terminate_instance |
| | self._delete_instance(context, instance, bdms, quotas) |
| | File \"/usr/lib/python2.7/site-packages/nova/hooks.py\", line 131, in inner |
| | rv = f(*args, **kwargs) |
| | File \"/usr/lib/python2.7/site-packages/nova/compute/manager.py\", line 2482, in _delete_instance |
| | quotas.rollback() |
| | File \"/usr/lib/python2.7/site-packages/nova/openstack/common/excutils.py\", line 82, in __exit__ |
| | six.reraise(self.type_, self.value, self.tb) |
| | File \"/usr/lib/python2.7/site-packages/nova/compute/manager.py\", line 2459, in _delete_instance |
| | self._shutdown_instance(context, instance, bdms) |
| | File \"/usr/lib/python2.7/site-packages/nova/compute/manager.py\", line 2389, in _shutdown_instance |
| | requested_networks) |
| | File \"/usr/lib/python2.7/site-packages/nova/openstack/common/excutils.py\", line 82, in __exit__ |
| | six.reraise(self.type_, self.value, self.tb) |
| | File \"/usr/lib/python2.7/site-packages/nova/compute/manager.py\", line 2378, in _shutdown_instance |
| | block_device_info) |
| | File \"/usr/lib/python2.7/site-packages/nova/virt/libvirt/driver.py\", line 1058, in destroy |
| | destroy_disks, migrate_data) |
| | File \"/usr/lib/python2.7/site-packages/nova/virt/libvirt/driver.py\", line 1173, in cleanup |
| | self._cleanup_rbd(instance) |
| | File \"/usr/lib/python2.7/site-packages/nova/virt/libvirt/driver.py\", line 1218, in _cleanup_rbd |
| | LibvirtDriver._get_rbd_driver().cleanup_volumes(instance) |
| | File \"/usr/lib/python2.7/site-packages/nova/virt/libvirt/rbd_utils.py\", line 266, in cleanup_volumes |
| | rbd.RBD().remove(client.ioctx, volume) |
| | File \"/usr/lib/python2.7/site-packages/rbd.py\", line 300, in remove |
| | raise make_ex(ret, 'error removing image') |
| | ", "created": "2015-03-25T14:17:14Z"}

in nova-compute.log:

2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 314, in decorated_function
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher return function(self, context, *args, **kwargs)
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 2525, in terminate_instance
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher do_terminate_instance(instance, bdms)
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.7/site-packages/nova/openstack/common/lockutils.py", line 272, in inner
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher return f(*args, **kwargs)
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 2523, in do_terminate_instance
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher self._set_instance_error_state(context, instance)
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.7/site-packages/nova/openstack/common/excutils.py", line 82, in __exit__
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher six.reraise(self.type_, self.value, self.tb)
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 2513, in do_terminate_instance
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher self._delete_instance(context, instance, bdms, quotas)
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.7/site-packages/nova/hooks.py", line 131, in inner
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher rv = f(*args, **kwargs)
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 2482, in _delete_instance
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher quotas.rollback()
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.7/site-packages/nova/openstack/common/excutils.py", line 82, in __exit__
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher six.reraise(self.type_, self.value, self.tb)
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 2459, in _delete_instance
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher self._shutdown_instance(context, instance, bdms)
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 2389, in _shutdown_instance
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher requested_networks)
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.7/site-packages/nova/openstack/common/excutils.py", line 82, in __exit__
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher six.reraise(self.type_, self.value, self.tb)
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 2378, in _shutdown_instance
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher block_device_info)
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.7/site-packages/nova/virt/libvirt/driver.py", line 1058, in destroy
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher destroy_disks, migrate_data)
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.7/site-packages/nova/virt/libvirt/driver.py", line 1173, in cleanup
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher self._cleanup_rbd(instance)
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.7/site-packages/nova/virt/libvirt/driver.py", line 1218, in _cleanup_rbd
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher LibvirtDriver._get_rbd_driver().cleanup_volumes(instance)
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.7/site-packages/nova/virt/libvirt/rbd_utils.py", line 266, in cleanup_volumes
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher rbd.RBD().remove(client.ioctx, volume)
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.7/site-packages/rbd.py", line 300, in remove
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher raise make_ex(ret, 'error removing image')
2015-03-25 10:17:14.951 15287 TRACE oslo.messaging.rpc.dispatcher ImageBusy: error removing image