Hi
My setup info: Ubuntu 20.04 LTS + OpenStack Yoga Charm base bundle (HA setup).
Randomly some instance is failing with the below error message.
Cinder VIP 10.91.8.24 is pinging and also able to do telnet to 10.91.8.24:8776.
In juju status, all workloads are in ACTIVE state. No errors.
Error Message:
Traceback (most recent call last): File "/usr/lib/python3/dist-packages/urllib3/connection.py", line 159, in _new_conn conn = connection.create_connection( File "/usr/lib/python3/dist-packages/urllib3/util/connection.py", line 84, in create_connection
Traceback (most recent call last): File "/usr/lib/python3/dist-packages/urllib3/connection.py", line 159, in _new_conn conn = connection.create_connection( File "/usr/lib/python3/dist-packages/urllib3/util/connection.py", line 84, in create_connection raise err File "/usr/lib/python3/dist-packages/urllib3/util/connection.py", line 74, in create_connection sock.connect(sa) File "/usr/lib/python3/dist-packages/eventlet/greenio/base.py", line 256, in connect socket_checkerr(fd) File "/usr/lib/python3/dist-packages/eventlet/greenio/base.py", line 54, in socket_checkerr raise socket.error(err, errno.errorcode[err]) OSError: [Errno 113] EHOSTUNREACH During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 665, in urlopen httplib_response = self._make_request( File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 376, in _make_request self._validate_conn(conn) File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 996, in _validate_conn conn.connect() File "/usr/lib/python3/dist-packages/urllib3/connection.py", line 314, in connect conn = self._new_conn() File "/usr/lib/python3/dist-packages/urllib3/connection.py", line 171, in _new_conn raise NewConnectionError( urllib3.exceptions.NewConnectionError: <urllib3.connection.VerifiedHTTPSConnection object at 0x7f13787160a0>: Failed to establish a new connection: [Errno 113] EHOSTUNREACH During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/usr/lib/python3/dist-packages/requests/adapters.py", line 439, in send resp = conn.urlopen( File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 719, in urlopen retries = retries.increment( File "/usr/lib/python3/dist-packages/urllib3/util/retry.py", line 436, in increment raise MaxRetryError(_pool, url, error or ResponseError(cause)) urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='10.91.8.24', port=8776): Max retries exceeded with url: / (Caused by NewConnectionError('<urllib3.connection.VerifiedHTTPSConnection object at 0x7f13787160a0>: Failed to establish a new connection: [Errno 113] EHOSTUNREACH')) During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/usr/lib/python3/dist-packages/keystoneauth1/session.py", line 1022, in _send_request resp = self.session.request(method, url, **kwargs) File "/usr/lib/python3/dist-packages/requests/sessions.py", line 542, in request resp = self.send(prep, **send_kwargs) File "/usr/lib/python3/dist-packages/requests/sessions.py", line 655, in send r = adapter.send(request, **kwargs) File "/usr/lib/python3/dist-packages/requests/adapters.py", line 516, in send raise ConnectionError(e, request=request) requests.exceptions.ConnectionError: HTTPSConnectionPool(host='10.91.8.24', port=8776): Max retries exceeded with url: / (Caused by NewConnectionError('<urllib3.connection.VerifiedHTTPSConnection object at 0x7f13787160a0>: Failed to establish a new connection: [Errno 113] EHOSTUNREACH')) During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/usr/lib/python3/dist-packages/nova/volume/cinder.py", line 395, in wrapper res = method(self, ctx, *args, **kwargs) File "/usr/lib/python3/dist-packages/nova/volume/cinder.py", line 433, in wrapper res = method(self, ctx, volume_id, *args, **kwargs) File "/usr/lib/python3/dist-packages/nova/volume/cinder.py", line 502, in get item = cinderclient( File "/usr/lib/python3/dist-packages/nova/volume/cinder.py", line 265, in cinderclient version = _check_microversion(context, url, microversion) File "/usr/lib/python3/dist-packages/nova/volume/cinder.py", line 183, in _check_microversion max_api_version = _get_highest_client_server_version(context, url) File "/usr/lib/python3/dist-packages/nova/volume/cinder.py", line 165, in _get_highest_client_server_version min_server, max_server = _get_server_version(context, url) File "/usr/lib/python3/dist-packages/nova/volume/cinder.py", line 144, in _get_server_version response = _SESSION.get(version_url, auth=auth) File "/usr/lib/python3/dist-packages/keystoneauth1/session.py", line 1141, in get return self.request(url, 'GET', **kwargs) File "/usr/lib/python3/dist-packages/keystoneauth1/session.py", line 931, in request resp = send(**kwargs) File "/usr/lib/python3/dist-packages/keystoneauth1/session.py", line 1038, in _send_request raise exceptions.ConnectFailure(msg) keystoneauth1.exceptions.connection.ConnectFailure: Unable to establish connection to https://10.91.8.24:8776/: HTTPSConnectionPool(host='10.91.8.24', port=8776): Max retries exceeded with url: / (Caused by NewConnectionError('<urllib3.connection.VerifiedHTTPSConnection object at 0x7f13787160a0>: Failed to establish a new connection: [Errno 113] EHOSTUNREACH')) During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/usr/lib/python3/dist-packages/nova/compute/manager.py", line 2059, in _prep_block_device driver_block_device.attach_block_devices( File "/usr/lib/python3/dist-packages/nova/virt/block_device.py", line 871, in attach_block_devices _log_and_attach(device) File "/usr/lib/python3/dist-packages/nova/virt/block_device.py", line 868, in _log_and_attach bdm.attach(*attach_args, **attach_kwargs) File "/usr/lib/python3/dist-packages/nova/virt/block_device.py", line 46, in wrapped ret_val = method(obj, context, *args, **kwargs) File "/usr/lib/python3/dist-packages/nova/virt/block_device.py", line 662, in attach volume = self._get_volume(context, volume_api, self.volume_id) File "/usr/lib/python3/dist-packages/nova/virt/block_device.py", line 341, in _get_volume return volume_api.get(context, volume_id, microversion='3.48') File "/usr/lib/python3/dist-packages/nova/volume/cinder.py", line 399, in wrapper _reraise(exception.CinderConnectionFailed(reason=err_msg)) File "/usr/lib/python3/dist-packages/nova/volume/cinder.py", line 486, in _reraise raise desired_exc.with_traceback(sys.exc_info()[2]) File "/usr/lib/python3/dist-packages/nova/volume/cinder.py", line 395, in wrapper res = method(self, ctx, *args, **kwargs) File "/usr/lib/python3/dist-packages/nova/volume/cinder.py", line 433, in wrapper res = method(self, ctx, volume_id, *args, **kwargs) File "/usr/lib/python3/dist-packages/nova/volume/cinder.py", line 502, in get item = cinderclient( File "/usr/lib/python3/dist-packages/nova/volume/cinder.py", line 265, in cinderclient version = _check_microversion(context, url, microversion) File "/usr/lib/python3/dist-packages/nova/volume/cinder.py", line 183, in _check_microversion max_api_version = _get_highest_client_server_version(context, url) File "/usr/lib/python3/dist-packages/nova/volume/cinder.py", line 165, in _get_highest_client_server_version min_server, max_server = _get_server_version(context, url) File "/usr/lib/python3/dist-packages/nova/volume/cinder.py", line 144, in _get_server_version response = _SESSION.get(version_url, auth=auth) File "/usr/lib/python3/dist-packages/keystoneauth1/session.py", line 1141, in get return self.request(url, 'GET', **kwargs) File "/usr/lib/python3/dist-packages/keystoneauth1/session.py", line 931, in request resp = send(**kwargs) File "/usr/lib/python3/dist-packages/keystoneauth1/session.py", line 1038, in _send_request raise exceptions.ConnectFailure(msg) nova.exception.CinderConnectionFailed: Connection to cinder host failed: Unable to establish connection to https://10.91.8.24:8776/: HTTPSConnectionPool(host='10.91.8.24', port=8776): Max retries exceeded with url: / (Caused by NewConnectionError('<urllib3.connection.VerifiedHTTPSConnection object at 0x7f13787160a0>: Failed to establish a new connection: [Errno 113] EHOSTUNREACH')) During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/usr/lib/python3/dist-packages/nova/compute/manager.py", line 2707, in _build_resources block_device_info = self._prep_block_device(context, instance, File "/usr/lib/python3/dist-packages/nova/compute/manager.py", line 2078, in _prep_block_device raise exception.InvalidBDM(str(ex)) nova.exception.InvalidBDM: Connection to cinder host failed: Unable to establish connection to https://10.91.8.24:8776/: HTTPSConnectionPool(host='10.91.8.24', port=8776): Max retries exceeded with url: / (Caused by NewConnectionError('<urllib3.connection.VerifiedHTTPSConnection object at 0x7f13787160a0>: Failed to establish a new connection: [Errno 113] EHOSTUNREACH')) During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/usr/lib/python3/dist-packages/nova/compute/manager.py", line 2312, in _do_build_and_run_instance self._build_and_run_instance(context, instance, image, File "/usr/lib/python3/dist-packages/nova/compute/manager.py", line 2521, in _build_and_run_instance compute_utils.notify_about_instance_create( File "/usr/lib/python3/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ self.force_reraise() File "/usr/lib/python3/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise raise self.value File "/usr/lib/python3/dist-packages/nova/compute/manager.py", line 2472, in _build_and_run_instance with self._build_resources(context, instance, File "/usr/lib/python3.8/contextlib.py", line 113, in __enter__ return next(self.gen) File "/usr/lib/python3/dist-packages/nova/compute/manager.py", line 2717, in _build_resources raise exception.BuildAbortException(instance_uuid=instance.uuid, nova.exception.BuildAbortException: Build of instance 5ec9e5f6-346b-45b6-b49e-b4a575ca8ba1 aborted: Connection to cinder host failed: Unable to establish connection to https://10.91.8.24:8776/: HTTPSConnectionPool(host='10.91.8.24', port=8776): Max retries exceeded with url: / (Caused by NewConnectionError('<urllib3.connection.VerifiedHTTPSConnection object at 0x7f13787160a0>: Failed to establish a new connection: [Errno 113] EHOSTUNREACH'))