Contaiers fail to start on undercloud VM and appears to be stuck on "sudo -E kolla_set_configs" or Z state

Solution In Progress - Updated -

Issue

  • Containers fail to start on undercloud server and appears to be stuck on "sudo -E kolla_set_configs", "/usr/bin/runc init" or Z state:
root      311912  2.6  0.2 627316 47124 ?        Ssl  08:22   0:00 /usr/bin/podman start heat_api
root      311916  3.8  0.0      0     0 ?        Zsl  08:22   0:00 [podman] <defunct>
root      311957  0.0  0.0 143472  2676 ?        Ssl  08:22   0:00 /usr/bin/conmon --api-version 1 -s -c aadc5382f23edecb0201369a2e9026c87d9dda6222b65935e68ca2e5a850f6c7 -u aadc5382f23edecb0201369a2e9026c87d9dda6222b65935e68ca2e5a850f6c7 
root      312756  0.0  0.3 553840 52044 ?        Rl   08:22   0:00  \_ /usr/bin/podman --root /var/lib/containers/storage --runroot /var/run/containers/storage --log-level error --cgroup-manager systemd --tmpdir /var/run/libpod --runtime 
root      311985  0.0  0.0 143472  2696 ?        Ssl  08:22   0:00 /usr/bin/conmon --api-version 1 -s -c 17c073a46a2835a4681ebffe537a70be2a2eed0fd402956c8e18eb81f4ce286c -u 17c073a46a2835a4681ebffe537a70be2a2eed0fd402956c8e18eb81f4ce286c 
root      312728  0.0  0.2 553584 45148 ?        Rl   08:22   0:00  \_ /usr/bin/podman --root /var/lib/containers/storage --runroot /var/run/containers/storage --log-level error --cgroup-manager systemd --tmpdir /var/run/libpod --runtime 
root      312001  3.4  0.3 861652 62292 ?        Ssl  08:22   0:00 /usr/bin/podman start zaqar_websocket
root      312006  0.0  0.0 143472  2540 ?        Ssl  08:22   0:00 /usr/bin/conmon --api-version 1 -s -c c282410c5ecfa092ab8e1e89069e5a03fff10c8e3e4e4ff915f19c5bf66d9b63 -u c282410c5ecfa092ab8e1e89069e5a03fff10c8e3e4e4ff915f19c5bf66d9b63 
root      312700  0.0  0.2 627572 44868 ?        Sl   08:22   0:00  \_ /usr/bin/podman --root /var/lib/containers/storage --runroot /var/run/containers/storage --log-level error --cgroup-manager systemd --tmpdir /var/run/libpod --runtime 
root      312009  0.0  0.0 143472  2692 ?        Ssl  08:22   0:00 /usr/bin/conmon --api-version 1 -s -c 820fddf4f308d77d25df1978186bbb03368df45991e673a06ad2b74d3d90346a -u 820fddf4f308d77d25df1978186bbb03368df45991e673a06ad2b74d3d90346a 
42445     312150  7.7  0.0   4204   696 ?        Ss   08:22   0:00  \_ dumb-init --single-child -- kolla_start
42445     312744  0.0  0.0  14920  3260 ?        S    08:22   0:00      \_ /bin/bash /usr/local/bin/kolla_start
root      312748  0.0  0.0 105072  6820 ?        R    08:22   0:00          \_ sudo -E kolla_set_configs
root      312158  4.5  0.3 935384 61764 ?        Ssl  08:22   0:00 /usr/bin/podman start mysql
root      312166  0.0  0.0 143472  2624 ?        Ssl  08:22   0:00 /usr/bin/conmon --api-version 1 -s -c eec13c7f3f0a867f4aa404d363de6a1e7695f1556bc599544fd859b7a8f59787 -u eec13c7f3f0a867f4aa404d363de6a1e7695f1556bc599544fd859b7a8f59787 
root      312167  1.7  0.1 785480 20920 ?        Sl   08:22   0:00  \_ /usr/bin/runc --systemd-cgroup --log-format=json --log /var/run/containers/storage/overlay-containers/eec13c7f3f0a867f4aa404d363de6a1e7695f1556bc599544fd859b7a8f59787/
root      312340  9.6  0.0 562828 15460 ?        Ssl  08:22   0:00      \_ /usr/bin/runc init
root      312206  6.3  0.3 935640 60500 ?        Ssl  08:22   0:00 /usr/bin/podman start mistral_api
root      312280  0.0  0.0 143472  2704 ?        Ssl  08:22   0:00 /usr/bin/conmon --api-version 1 -s -c 62e39d32e9c81b41292f8cc0a7624d2508e66b2dafcd50aa8f50fb25aee1f88e -u 62e39d32e9c81b41292f8cc0a7624d2508e66b2dafcd50aa8f50fb25aee1f88e 
root      312281  2.3  0.1 785736 20916 ?        Sl   08:22   0:00  \_ /usr/bin/runc --systemd-cgroup --log-format=json --log /var/run/containers/storage/overlay-containers/62e39d32e9c81b41292f8cc0a7624d2508e66b2dafcd50aa8f50fb25aee1f88e/
root      312396 14.0  0.0 489096 15112 ?        Ssl  08:22   0:00      \_ /usr/bin/runc init
root      312284  0.0  0.0 143472  2628 ?        Ssl  08:22   0:00 /usr/bin/conmon --api-version 1 -s -c 6eb02087e068438aa0a37599e035f94db9f0a2cd5e881af8539d084c9366cce6 -u 6eb02087e068438aa0a37599e035f94db9f0a2cd5e881af8539d084c9366cce6 
42436     312368 14.5  0.0   4204   808 ?        Ss   08:22   0:00  \_ dumb-init --single-child -- kolla_start
42436     312736  0.0  0.0  14920  3388 ?        S    08:22   0:00      \_ /bin/bash /usr/local/bin/kolla_start
root      312740  0.0  0.0 105072  6760 ?        S    08:22   0:00          \_ sudo -E kolla_set_configs
root      312752  0.0  0.0  24972  7908 ?        R    08:22   0:00              \_ python3 /usr/local/bin/kolla_set_configs
root      312308  6.3  0.3 935384 62864 ?        Ssl  08:22   0:00 /usr/bin/podman start swift_proxy
root      312311  0.0  0.0 143472  2572 ?        Ssl  08:22   0:00 /usr/bin/conmon --api-version 1 -s -c 1cbe224cfc2ba0d61a948f373dd7aec0dab371d53aa7a4a9579abdb9d6ea9b8c -u 1cbe224cfc2ba0d61a948f373dd7aec0dab371d53aa7a4a9579abdb9d6ea9b8c 
42418     312367 15.0  0.0   4204   708 ?        Ss   08:22   0:00  \_ dumb-init --single-child -- kolla_start
42418     312776  0.0  0.0  14920  3220 ?        S    08:22   0:00      \_ /bin/bash /usr/local/bin/kolla_start
root      312784  0.0  0.0 105072  6780 ?        R    08:22   0:00          \_ sudo -E kolla_set_configs
root      312363 12.5  0.3 1009116 60952 ?       Ssl  08:22   0:00 /usr/bin/podman start swift_object_server
root      312421  0.0  0.0 143472  2732 ?        Ssl  08:22   0:00 /usr/bin/conmon --api-version 1 -s -c e2ef4a5c8cbca2aecb5db6a119a73a580bcfb58ef71614b985236d20680a936a -u e2ef4a5c8cbca2aecb5db6a119a73a580bcfb58ef71614b985236d20680a936a 
root      312422  3.0  0.1 711748 18668 ?        Sl   08:22   0:00  \_ /usr/bin/runc --systemd-cgroup --log-format=json --log /var/run/containers/storage/overlay-containers/e2ef4a5c8cbca2aecb5db6a119a73a580bcfb58ef71614b985236d20680a936a/
root      312540 25.0  0.0 489096 13152 ?        Rsl  08:22   0:00      \_ /usr/bin/runc init
root      312501  7.0  0.2 553840 44876 ?        Rsl  08:22   0:00 /usr/bin/podman start ironic_api
root      312502 18.0  0.3 1083104 64512 ?       Ssl  08:22   0:00 /usr/bin/podman start swift_container_updater
root      312548  0.0  0.0 143472  2728 ?        Ssl  08:22   0:00 /usr/bin/conmon --api-version 1 -s -c dd3d79faa973fbce79a1314f5d5be732ddd492eb81b80200e28e3ba05a0bce4a -u dd3d79faa973fbce79a1314f5d5be732ddd492eb81b80200e28e3ba05a0bce4a 
root      312550  5.0  0.1 785736 20748 ?        Sl   08:22   0:00  \_ /usr/bin/runc --systemd-cgroup --log-format=json --log /var/run/containers/storage/overlay-containers/dd3d79faa973fbce79a1314f5d5be732ddd492eb81b80200e28e3ba05a0bce4a/
root      312636  5.0  0.0 562688 12952 ?        Rsl  08:22   0:00      \_ /usr/bin/runc init
root      312565 19.0  0.3 1009372 62692 ?       Ssl  08:22   0:00 /usr/bin/podman start keystone
root      312576  0.0  0.0 143472  2680 ?        Ssl  08:22   0:00 /usr/bin/conmon --api-version 1 -s -c aca689966416bfcb0239f810965e592b4aa433e64dbbb889bd8526423292d9dc -u aca689966416bfcb0239f810965e592b4aa433e64dbbb889bd8526423292d9dc 
root      312577  7.0  0.1 859212 20552 ?        Sl   08:22   0:00  \_ /usr/bin/runc --systemd-cgroup --log-format=json --log /var/run/containers/storage/overlay-containers/aca689966416bfcb0239f810965e592b4aa433e64dbbb889bd8526423292d9dc/
root      312638  8.0  0.0 489088 15104 ?        Rsl  08:22   0:00      \_ /usr/bin/runc init
root      312623  8.0  0.3 714040 51508 ?        Rsl  08:22   0:00 /usr/bin/podman start swift_container_server
root      312707  0.0  0.0 143472  2580 ?        Ssl  08:22   0:00 /usr/bin/conmon --api-version 1 -s -c 2357c8f046ab51111a9f7be692f15ab92789e7be6f686d70235bf1c683e4571b -u 2357c8f046ab51111a9f7be692f15ab92789e7be6f686d70235bf1c683e4571b 
root      312708  0.0  0.1 563964 16836 ?        Sl   08:22   0:00  \_ /usr/bin/runc --systemd-cgroup --log-format=json --log /var/run/containers/storage/overlay-containers/2357c8f046ab51111a9f7be692f15ab92789e7be6f686d70235bf1c683e4571b/
root      312710  0.0  0.0 143472  2544 ?        Ssl  08:22   0:00 /usr/bin/conmon --api-version 1 -s -c dc283c1eaca143e06944aa5e1d23d72356c9a489f521bbd09ea0bf9822470627 -u dc283c1eaca143e06944aa5e1d23d72356c9a489f521bbd09ea0bf9822470627 
root      312711  0.0  0.1 637696 18776 ?        Rl   08:22   0:00  \_ /usr/bin/runc --systemd-cgroup --log-format=json --log /var/run/containers/storage/overlay-containers/dc283c1eaca143e06944aa5e1d23d72356c9a489f521bbd09ea0bf9822470627/
root      312755  0.0  0.2 412908 42880 ?        Rsl  08:22   0:00 /usr/bin/podman start placement_api
root      312763  0.0  0.3 713784 51848 ?        Rsl  08:22   0:00 /usr/bin/podman start nova_api
root      312769  0.0  0.1 175988 31600 ?        Rs   08:22   0:00 /usr/bin/podman start swift_rsync

Environment

  • Red Hat OpenStack Platform 16.0 (RHOSP)

Subscriber exclusive content

A Red Hat subscription provides unlimited access to our knowledgebase of over 48,000 articles and solutions.

Current Customers and Partners

Log in for full access

Log In