Nova instance creation fail after upgrading from liberty to mitaka

Issue

Creating instance failed with below message after upgrading from OSP8 (liberty) to OSP9 (mitaka)

There is no request_specs table in nova database.

2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions [req-c594c8d9-50b4-48d8-868c-f738221b4ed6 7a02b2d49d8142fa914f3c5016a258e9 f68211f3b0384ddfae0ed050c71c5494 - - -] Unexpected exception in API method
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions Traceback (most recent call last):
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions   File "/usr/lib/python2.7/site-packages/nova/api/openstack/extensions.py", line 478, in wrapped
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions     return f(*args, **kwargs)
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions   File "/usr/lib/python2.7/site-packages/nova/api/validation/__init__.py", line 73, in wrapper
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions     return func(*args, **kwargs)
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions   File "/usr/lib/python2.7/site-packages/nova/api/validation/__init__.py", line 73, in wrapper
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions     return func(*args, **kwargs)
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions   File "/usr/lib/python2.7/site-packages/nova/api/validation/__init__.py", line 73, in wrapper
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions     return func(*args, **kwargs)
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions   File "/usr/lib/python2.7/site-packages/nova/api/openstack/compute/servers.py", line 629, in create
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions     **create_kwargs)
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions   File "/usr/lib/python2.7/site-packages/nova/hooks.py", line 154, in inner
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions     rv = f(*args, **kwargs)
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions   File "/usr/lib/python2.7/site-packages/nova/compute/api.py", line 1564, in create
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions     check_server_group_quota=check_server_group_quota)
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions   File "/usr/lib/python2.7/site-packages/nova/compute/api.py", line 1177, in _create_instance
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions     instance_group, check_server_group_quota, filter_properties)
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions   File "/usr/lib/python2.7/site-packages/nova/compute/api.py", line 1035, in _provision_instances
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions     quotas.rollback()
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions   File "/usr/lib/python2.7/site-packages/oslo_utils/excutils.py", line 220, in __exit__
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions     self.force_reraise()
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions   File "/usr/lib/python2.7/site-packages/oslo_utils/excutils.py", line 196, in force_reraise
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions     six.reraise(self.type_, self.value, self.tb)
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions   File "/usr/lib/python2.7/site-packages/nova/compute/api.py", line 977, in _provision_instances
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions     req_spec.create()
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions   File "/usr/lib/python2.7/site-packages/oslo_versionedobjects/base.py", line 223, in wrapper
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions     return fn(self, *args, **kwargs)
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions   File "/usr/lib/python2.7/site-packages/nova/objects/request_spec.py", line 432, in create
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions     db_spec = self._create_in_db(self._context, updates)
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions   File "/usr/lib/python2.7/site-packages/oslo_db/sqlalchemy/enginefacade.py", line 709, in wrapper
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions     return fn(*args, **kwargs)
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions   File "/usr/lib/python2.7/site-packages/nova/objects/request_spec.py", line 403, in _create_in_db
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions     db_spec.save(context.session)
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions   File "/usr/lib/python2.7/site-packages/oslo_db/sqlalchemy/models.py", line 48, in save
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions     session.flush()
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions   File "/usr/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 2027, in flush
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions     self._flush(objects)
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions   File "/usr/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 2145, in _flush
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions     transaction.rollback(_capture_exception=True)
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions   File "/usr/lib64/python2.7/site-packages/sqlalchemy/util/langhelpers.py", line 60, in __exit__
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions     compat.reraise(exc_type, exc_value, exc_tb)
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions   File "/usr/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 2109, in _flush
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions     flush_context.execute()
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions   File "/usr/lib64/python2.7/site-packages/sqlalchemy/orm/unitofwork.py", line 373, in execute
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions     rec.execute(self)
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions   File "/usr/lib64/python2.7/site-packages/sqlalchemy/orm/unitofwork.py", line 532, in execute
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions     uow
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions   File "/usr/lib64/python2.7/site-packages/sqlalchemy/orm/persistence.py", line 174, in save_obj
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions     mapper, table, insert)
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions   File "/usr/lib64/python2.7/site-packages/sqlalchemy/orm/persistence.py", line 800, in _emit_insert_statements
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions     execute(statement, params)
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions   File "/usr/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 914, in execute
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions     return meth(self, multiparams, params)
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions   File "/usr/lib64/python2.7/site-packages/sqlalchemy/sql/elements.py", line 323, in _execute_on_connection
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions     return connection._execute_clauseelement(self, multiparams, params)
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions   File "/usr/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1010, in _execute_clauseelement
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions     compiled_sql, distilled_params
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions   File "/usr/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1146, in _execute_context
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions     context)
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions   File "/usr/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1337, in _handle_dbapi_exception
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions     util.raise_from_cause(newraise, exc_info)
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions   File "/usr/lib64/python2.7/site-packages/sqlalchemy/util/compat.py", line 200, in raise_from_cause
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions     reraise(type(exception), exception, tb=exc_tb)
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions   File "/usr/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1139, in _execute_context
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions     context)
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions   File "/usr/lib64/python2.7/site-packages/sqlalchemy/engine/default.py", line 450, in do_execute
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions     cursor.execute(statement, parameters)
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions   File "/usr/lib64/python2.7/site-packages/MySQLdb/cursors.py", line 205, in execute
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions     self.errorhandler(self, exc, value)
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions   File "/usr/lib64/python2.7/site-packages/MySQLdb/connections.py", line 36, in defaulterrorhandler
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions     raise errorclass, errorvalue
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions ProgrammingError: (_mysql_exceptions.ProgrammingError) (1146, "Table 'nova.request_specs' doesn't exist") [SQL: u'INSERT INTO request_specs (created_at, updated_at, instance_uuid, spec) VALUES (%s, %s, %s, %s)'] [parameters: (datetime.datetime(2017, 10, 23, 7, 37, 39, 307959), None, 'b843fae8-9e79-40b6-ac59-6cba5d81a55e', '{"nova_object.version": "1.5", "nova_object.changes": ["instance_uuid", "retry", "num_instances", "pci_requests", "limits", "availability_zone", "force_nodes", "image", "instance_group", "force_hosts", "numa_topology", "flavor", "project_id", "scheduler_hints", "ignore_hosts"], "nova_object.name": "RequestSpec", "nova_object.data": {"instance_uuid": "b843fae8-9e79-40b6-ac59-6cba5d81a55e", "retry": null, "num_instances": 1, "pci_requests": {"nova_object.version": "1.1", "nova_object.changes": ["requests"], "nova_object.name": "InstancePCIRequests", "nova_object.data": {"requests": []}, "nova_object.namespace": "nova"}, "limits": {"nova_object.version": "1.0", "nova_object.changes": ["vcpu", "memory_mb", "numa_topology", "disk_gb"], "nova_object.name": "SchedulerLimits", "nova_object.data": {"vcpu": null, "memory_mb": null, "disk_gb": null, "numa_topology": null}, "nova_object.namespace": "nova"}, "availability_zone": "nova", "force_nodes": null, "image": {"nova_object.version": "1.8", "nova_object.changes": ["min_disk", "status", "min_ram", "properties", "size"], "nova_object.name": "ImageMeta", "nova_object.data": {"status": "active", "min_disk": 0, "min_ram": 0, "properties": {"nova_object.version": "1.12", "nova_object.name": "ImageMetaProps", "nova_object.data": {}, "nova_object.namespace": "nova"}, "size": 21474836480}, "nova_object.namespace": "nova"}, "instance_group": null, "force_hosts": null, "numa_topology": null, "ignore_hosts": null, "flavor": {"nova_object.version": "1.1", "nova_object.name": "Flavor", "nova_object.data": {"disabled": false, "root_gb": 20, "name": "m1.small", "flavorid": "2", "deleted": false, "created_at": null, "ephemeral_gb": 0, "updated_at": null, "memory_mb": 2048, "vcpus": 1, "extra_specs": {}, "swap": 0, "rxtx_factor": 1.0, "is_public": true, "deleted_at": null, "vcpu_weight": 0, "id": 13}, "nova_object.namespace": "nova"}, "project_id": "f68211f3b0384ddfae0ed050c71c5494", "scheduler_hints": {}}, "nova_object.namespace": "nova"}')]
2017-10-23 16:37:39.336 15000 ERROR nova.api.openstack.extensions 
2017-10-23 16:37:39.490 15000 INFO nova.api.openstack.wsgi [req-c594c8d9-50b4-48d8-868c-f738221b4ed6 7a02b2d49d8142fa914f3c5016a258e9 f68211f3b0384ddfae0ed050c71c5494 - - -] HTTP exception thrown: Unexpected API Error. Please report this at http://bugs.launchpad.net/nova/ and attach the Nova API log if possible.
<class 'sqlalchemy.exc.ProgrammingError'>