NooBaa Endpoint pod goes into CrashLoopBackOff state after upgrading to ODF v4.9
Issue
Uploading data to buckets backed by Thanos caused the noobaa-endpoint pod to go into CLBO. This results in the following crash in the noobaa-endpoint logs:
Dec-30 12:26:45.838 [Endpoint/14] [L0] core.sdk.object_io:: UPLOAD: { obj_id: '61cda585026af6000ea2466f', bucket: 'first.bucket', key: 'debug/metas/01FR3B94Q3CJQ3RFWR32FRYJKM.json' } streaming to first.bucket debug/metas/01FR3B94Q3CJQ3RFWR32FRYJKM.json
Dec-30 12:26:45.840 [Endpoint/14] [ERROR] core.sdk.object_io:: _upload_stream error Error: Pipeline called on destroyed stream
at Object.pipeline (/root/node_modules/noobaa-core/src/util/stream_utils.js:36:21)
at ObjectIO._upload_stream_internal (/root/node_modules/noobaa-core/src/sdk/object_io.js:435:28)
at /root/node_modules/noobaa-core/src/sdk/object_io.js:365:28
at Semaphore.surround_count (/root/node_modules/noobaa-core/src/util/semaphore.js:90:90)
at async ObjectIO._upload_stream (/root/node_modules/noobaa-core/src/sdk/object_io.js:363:25)
at async ObjectIO.upload_object (/root/node_modules/noobaa-core/src/sdk/object_io.js:233:17)
at async NamespaceNB.upload_object (/root/node_modules/noobaa-core/src/sdk/namespace_nb.js:126:23)
at async ObjectSDK.upload_object (/root/node_modules/noobaa-core/src/sdk/object_sdk.js:504:23)
at async Object.put_object [as handler] (/root/node_modules/noobaa-core/src/endpoint/s3/ops/s3_put_object.js:30:19)
at async handle_request (/root/node_modules/noobaa-core/src/endpoint/s3/s3_rest.js:149:19) Error: Pipeline called on destroyed stream
at Object.pipeline (/root/node_modules/noobaa-core/src/util/stream_utils.js:36:21)
at ObjectIO._upload_stream_internal (/root/node_modules/noobaa-core/src/sdk/object_io.js:435:28)
at /root/node_modules/noobaa-core/src/sdk/object_io.js:365:28
at Semaphore.surround_count (/root/node_modules/noobaa-core/src/util/semaphore.js:90:90)
at async ObjectIO._upload_stream (/root/node_modules/noobaa-core/src/sdk/object_io.js:363:25)
at async ObjectIO.upload_object (/root/node_modules/noobaa-core/src/sdk/object_io.js:233:17)
at async NamespaceNB.upload_object (/root/node_modules/noobaa-core/src/sdk/namespace_nb.js:126:23)
at async ObjectSDK.upload_object (/root/node_modules/noobaa-core/src/sdk/object_sdk.js:504:23)
at async Object.put_object [as handler] (/root/node_modules/noobaa-core/src/endpoint/s3/ops/s3_put_object.js:30:19)
at async handle_request (/root/node_modules/noobaa-core/src/endpoint/s3/s3_rest.js:149:19)
Dec-30 12:26:45.840 [Endpoint/14] [WARN] core.sdk.object_io:: upload_object: failed upload { bucket: 'first.bucket', key: 'debug/metas/01FR3B94Q3CJQ3RFWR32FRYJKM.json', md_conditions: undefined, obj_id: '61cda585026af6000ea2466f', size: 0, num_parts: 0 } Error: Pipeline called on destroyed stream
at Object.pipeline (/root/node_modules/noobaa-core/src/util/stream_utils.js:36:21)
at ObjectIO._upload_stream_internal (/root/node_modules/noobaa-core/src/sdk/object_io.js:435:28)
at /root/node_modules/noobaa-core/src/sdk/object_io.js:365:28
at Semaphore.surround_count (/root/node_modules/noobaa-core/src/util/semaphore.js:90:90)
at async ObjectIO._upload_stream (/root/node_modules/noobaa-core/src/sdk/object_io.js:363:25)
at async ObjectIO.upload_object (/root/node_modules/noobaa-core/src/sdk/object_io.js:233:17)
at async NamespaceNB.upload_object (/root/node_modules/noobaa-core/src/sdk/namespace_nb.js:126:23)
at async ObjectSDK.upload_object (/root/node_modules/noobaa-core/src/sdk/object_sdk.js:504:23)
at async Object.put_object [as handler] (/root/node_modules/noobaa-core/src/endpoint/s3/ops/s3_put_object.js:30:19)
at async handle_request (/root/node_modules/noobaa-core/src/endpoint/s3/s3_rest.js:149:19)
Dec-30 12:26:45.841 [Endpoint/14] [ERROR] CONSOLE:: PANIC: process uncaughtException Error: Pipeline called on destroyed stream
at Object.pipeline (/root/node_modules/noobaa-core/src/util/stream_utils.js:36:21)
at ObjectIO._upload_stream_internal (/root/node_modules/noobaa-core/src/sdk/object_io.js:435:28)
at /root/node_modules/noobaa-core/src/sdk/object_io.js:365:28
at Semaphore.surround_count (/root/node_modules/noobaa-core/src/util/semaphore.js:90:90)
at async ObjectIO._upload_stream (/root/node_modules/noobaa-core/src/sdk/object_io.js:363:25)
at async ObjectIO.upload_object (/root/node_modules/noobaa-core/src/sdk/object_io.js:233:17)
at async NamespaceNB.upload_object (/root/node_modules/noobaa-core/src/sdk/namespace_nb.js:126:23)
at async ObjectSDK.upload_object (/root/node_modules/noobaa-core/src/sdk/object_sdk.js:504:23)
at async Object.put_object [as handler] (/root/node_modules/noobaa-core/src/endpoint/s3/ops/s3_put_object.js:30:19)
at async handle_request (/root/node_modules/noobaa-core/src/endpoint/s3/s3_rest.js:149:19)
Environment
- Red Hat OpenShift Data Foundation 4.9
Subscriber exclusive content
A Red Hat subscription provides unlimited access to our knowledgebase, tools, and much more.