SIGSEGV error in RHOSP16.1 guests.

Issue

• Userspace applications running inside RHOSP16.1 guests quit with segfault or sigabrt, however the guest kernel does not crash.
• Core analysis suggests corrupt memory or hypervisor hardware issues, but the hypervisor hardware appears to be healthy.

• Issue does not present across all hypervisors; migrating the guest to another hypervisor may cause the issue to disappear.

• Log for the issue:

2022-08-02T07:40:59.903Z [INFO]  agent.server.raft: starting snapshot up to: index=120553641
    2022-08-02T07:40:59.903Z [INFO]  agent.server.snapshot: creating new snapshot: path=/data/consul/raft/snapshots/9-120553641-1659426059903.tmp
    2022-08-02T07:40:59.954Z [INFO]  agent.server.snapshot: reaping snapshot: path=/data/consul/raft/snapshots/9-120520000-1659423358618
    2022-08-02T07:40:59.954Z [INFO]  agent.server.raft: compacting logs: from=120526398 to=120543402
    2022-08-02T07:40:59.966Z [INFO]  agent.server.raft: snapshot complete up to: index=120553641
    2022-08-02T07:42:06.155Z [INFO]  agent: Synced check: check=service:platform.srv.fm.master-snmp-agent.ext_ipv6
    2022-08-02T07:42:09.711Z [INFO]  agent: Synced check: check=service:platform.srv.cm-gui.ext
    2022-08-02T07:43:03.409Z [INFO]  agent: Synced check: check=service:app.srv.pm-reporter
    2022-08-02T07:43:14.361Z [INFO]  agent: Synced check: check=service:app.srv.pm-reporter.ext
    2022-08-02T07:43:41.056Z [INFO]  agent: Synced check: check=service:platform.srv.cm-gui.ext_ipv6
    2022-08-02T07:43:53.544Z [INFO]  agent: Synced check: check=service:app.srv.pm-reporter.ext_ipv6
    2022-08-02T07:44:43.905Z [INFO]  agent: Synced check: check=service:platform.srv.sharedstorage.gluster-server-1
    2022-08-02T07:47:14.065Z [INFO]  agent: Synced check: check=service:platform.srv.fm.master-snmp-agent.ext
    2022-08-02T07:47:40.327Z [INFO]  agent: Synced check: check=service:platform.srv.fm.master-snmp-agent.ext_ipv6
unexpected fault address 0x18b0d593
fatal error: fault
[signal SIGSEGV: segmentation violation code=0x1 addr=0x18b0d593 pc=0x471565]

goroutine 97 [running]:
runtime.throw(0x33e1d65, 0x5)
            /usr/local/go/src/runtime/panic.go:1116 +0x72 fp=0xc00298cf00 sp=0xc00298ced0 pc=0x4378d2
runtime.sigpanic()
            /usr/local/go/src/runtime/signal_unix.go:749 +0x405 fp=0xc00298cf30 sp=0xc00298cf00 pc=0x44e3e5
runtime.memmove(0xc00385276a, 0x18b0d4ff, 0x114)
            /usr/local/go/src/runtime/memmove_amd64.s:332 +0x3c5 fp=0xc00298cf38 sp=0xc00298cf30 pc=0x471565
github.com/boltdb/bolt.(*node).write(0xc003a28770, 0xc003852000)
            /go/pkg/mod/github.com/boltdb/bolt@v1.3.1/node.go:241 +0x299 fp=0xc00298d0a0 sp=0xc00298cf38 pc=0x1934e19
github.com/boltdb/bolt.(*node).spill(0xc003a28770, 0xc00383a380, 0x1)
            /go/pkg/mod/github.com/boltdb/bolt@v1.3.1/node.go:378 +0x2c5 fp=0xc00298d190 sp=0xc00298d0a0 pc=0x1935a05
github.com/boltdb/bolt.(*node).spill(0xc003a28700, 0xc00383a360, 0x1)
            /go/pkg/mod/github.com/boltdb/bolt@v1.3.1/node.go:350 +0xba fp=0xc00298d280 sp=0xc00298d190 pc=0x19357fa
github.com/boltdb/bolt.(*node).spill(0xc003a28690, 0xc0050ce090, 0xc00298d4f0)
            /go/pkg/mod/github.com/boltdb/bolt@v1.3.1/node.go:350 +0xba fp=0xc00298d370 sp=0xc00298d280 pc=0x19357fa
github.com/boltdb/bolt.(*Bucket).spill(0xc00502edc0, 0xc003a17e00, 0xc00298d6e0)
            /go/pkg/mod/github.com/boltdb/bolt@v1.3.1/bucket.go:570 +0x497 fp=0xc00298d560 sp=0xc00298d370 pc=0x192bab7
github.com/boltdb/bolt.(*Bucket).spill(0xc001e30c58, 0x70c200ae48e, 0x5191aa0)
            /go/pkg/mod/github.com/boltdb/bolt@v1.3.1/bucket.go:537 +0x3f6 fp=0xc00298d750 sp=0xc00298d560 pc=0x192ba16
github.com/boltdb/bolt.(*Tx).Commit(0xc001e30c40, 0xc003594758, 0x8)
            /go/pkg/mod/github.com/boltdb/bolt@v1.3.1/tx.go:163 +0xfc fp=0xc00298d898 sp=0xc00298d750 pc=0x19387fc
github.com/hashicorp/raft-boltdb.(*BoltStore).StoreLogs(0xc0007f4000, 0xc003ce6458, 0x1, 0x1, 0x0, 0x0)
            /go/pkg/mod/github.com/hashicorp/raft-boltdb@v0.0.0-20171010151810-6e5ba93211ea/bolt_store.go:187 +0x285 fp=0xc00298d938 sp=0xc00298d898 pc=0x193e2c5
github.com/hashicorp/raft.(*LogCache).StoreLogs(0xc0007e6100, 0xc003ce6458, 0x1, 0x1, 0xc000b2ed20, 0x13)
            /go/pkg/mod/github.com/hashicorp/raft@v1.2.0/log_cache.go:61 +0xe6 fp=0xc00298d980 sp=0xc00298d938 pc=0x9b4e26
github.com/hashicorp/raft.(*Raft).appendEntries(0xc00038af00, 0x2c91680, 0xc0043d6f60, 0x0, 0x0, 0xc0043d6ea0, 0xc0043d6f60)
            /go/pkg/mod/github.com/hashicorp/raft@v1.2.0/raft.go:1354 +0x95e fp=0xc00298db18 sp=0xc00298d980 pc=0x9c4a1e
github.com/hashicorp/raft.(*Raft).processRPC(0xc00038af00, 0x2c91680, 0xc0043d6f60, 0x0, 0x0, 0xc0043d6ea0)
            /go/pkg/mod/github.com/hashicorp/raft@v1.2.0/raft.go:1216 +0x466 fp=0xc00298dbb8 sp=0xc00298db18 pc=0x9c3b26
github.com/hashicorp/raft.(*Raft).runFollower(0xc00038af00)
            /go/pkg/mod/github.com/hashicorp/raft@v1.2.0/raft.go:159 +0xf51 fp=0xc00298df60 sp=0xc00298dbb8 pc=0x9bb831
github.com/hashicorp/raft.(*Raft).run(0xc00038af00)
            /go/pkg/mod/github.com/hashicorp/raft@v1.2.0/raft.go:140 +0x86 fp=0xc00298df88 sp=0xc00298df60 pc=0x9ba8a6
github.com/hashicorp/raft.(*Raft).run-fm()
            /go/pkg/mod/github.com/hashicorp/raft@v1.2.0/raft.go:126 +0x2a fp=0xc00298dfa0 sp=0xc00298df88 pc=0x9d464a
github.com/hashicorp/raft.(*raftState).goFunc.func1(0xc00038af00, 0xc0007fc100)
            /go/pkg/mod/github.com/hashicorp/raft@v1.2.0/state.go:146 +0x55 fp=0xc00298dfd0 sp=0xc00298dfa0 pc=0x9d2995
runtime.goexit()
            /usr/local/go/src/runtime/asm_amd64.s:1374 +0x1 fp=0xc00298dfd8 sp=0xc00298dfd0 pc=0x4703a1
created by github.com/hashicorp/raft.(*raftState).goFunc
            /go/pkg/mod/github.com/hashicorp/raft@v1.2.0/state.go:144 +0x66

• Hypervisor kernel is 4.18.0-193.37.1.el8_2.x86_64