From 969c6a8a7be6c9eb20aee1add53d27a7fd5cde1f Mon Sep 17 00:00:00 2001 From: Liam Young Date: Fri, 1 Dec 2023 14:42:56 +0000 Subject: [PATCH 1/2] Make wait_for_endpoints more robust. As seen in Bug #2045206 wait_for_endpoints can fails if keystone is in a transient state. This change puts tenacity around the keystone client setup to make it more resilient. Closes-Bug: 2045206 --- zaza/openstack/charm_tests/keystone/setup.py | 25 ++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/zaza/openstack/charm_tests/keystone/setup.py b/zaza/openstack/charm_tests/keystone/setup.py index 2e09682d5..90647002e 100644 --- a/zaza/openstack/charm_tests/keystone/setup.py +++ b/zaza/openstack/charm_tests/keystone/setup.py @@ -192,6 +192,26 @@ def wait_for_url(url, ok_codes=None): assert r.status_code in ok_codes +def wait_for_client(): + """Wait for client to be returned successfully. + + If keystone is still in a transient state then it may take a few retries + before a client is returned + """ + for attempt in tenacity.Retrying( + stop=tenacity.stop_after_attempt(10), + wait=tenacity.wait_exponential( + multiplier=1, min=2, max=60)): + with attempt: + overcloud_auth = openstack_utils.get_overcloud_auth() + wait_for_url(overcloud_auth['OS_AUTH_URL']) + session = openstack_utils.get_overcloud_keystone_session() + keystone_client = openstack_utils.get_keystone_session_client( + session) + + return keystone_client + + def wait_for_all_endpoints(interface='public'): """Check all endpoints are returning an acceptable return code. @@ -199,10 +219,7 @@ def wait_for_all_endpoints(interface='public'): :type interface: str :raises: AssertionError """ - overcloud_auth = openstack_utils.get_overcloud_auth() - wait_for_url(overcloud_auth['OS_AUTH_URL']) - session = openstack_utils.get_overcloud_keystone_session() - keystone_client = openstack_utils.get_keystone_session_client(session) + keystone_client = wait_for_client() for service in keystone_client.services.list(): for ep in keystone_client.endpoints.list(service=service, interface=interface): From 7fe5b88a694dd756c7e9dc2d4f608e07dc9481ed Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 4 Dec 2023 13:18:08 +0000 Subject: [PATCH 2/2] Move wait_for_url and rename wait_for_client --- zaza/openstack/charm_tests/keystone/setup.py | 46 +------------------- zaza/openstack/utilities/openstack.py | 41 +++++++++++++++++ 2 files changed, 43 insertions(+), 44 deletions(-) diff --git a/zaza/openstack/charm_tests/keystone/setup.py b/zaza/openstack/charm_tests/keystone/setup.py index 90647002e..6134cdb1e 100644 --- a/zaza/openstack/charm_tests/keystone/setup.py +++ b/zaza/openstack/charm_tests/keystone/setup.py @@ -16,7 +16,6 @@ import logging import requests -import tenacity import keystoneauth1 @@ -171,47 +170,6 @@ def add_tempest_roles(): _add_additional_roles(TEMPEST_ROLES) -def wait_for_url(url, ok_codes=None): - """Wait for url to return acceptable return code. - - :param url: url to test - :type url: str - :param ok_codes: HTTP codes that are acceptable - :type ok_codes: Optional[List[int]] - :raises: AssertionError - """ - if not ok_codes: - ok_codes = [requests.codes.ok] - for attempt in tenacity.Retrying( - stop=tenacity.stop_after_attempt(10), - wait=tenacity.wait_exponential( - multiplier=1, min=2, max=60)): - with attempt: - r = requests.get(url) - logging.info("{} returned {}".format(url, r.status_code)) - assert r.status_code in ok_codes - - -def wait_for_client(): - """Wait for client to be returned successfully. - - If keystone is still in a transient state then it may take a few retries - before a client is returned - """ - for attempt in tenacity.Retrying( - stop=tenacity.stop_after_attempt(10), - wait=tenacity.wait_exponential( - multiplier=1, min=2, max=60)): - with attempt: - overcloud_auth = openstack_utils.get_overcloud_auth() - wait_for_url(overcloud_auth['OS_AUTH_URL']) - session = openstack_utils.get_overcloud_keystone_session() - keystone_client = openstack_utils.get_keystone_session_client( - session) - - return keystone_client - - def wait_for_all_endpoints(interface='public'): """Check all endpoints are returning an acceptable return code. @@ -219,11 +177,11 @@ def wait_for_all_endpoints(interface='public'): :type interface: str :raises: AssertionError """ - keystone_client = wait_for_client() + keystone_client = openstack_utils.get_keystone_overcloud_session_client() for service in keystone_client.services.list(): for ep in keystone_client.endpoints.list(service=service, interface=interface): - wait_for_url( + openstack_utils.wait_for_url( ep.url, # Heat cloudformation and orchestration return 400 and 401 [ diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index e3032d2c2..540b9a3c1 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -29,6 +29,7 @@ import paramiko import pathlib import re +import requests import shutil import six import subprocess @@ -3449,3 +3450,43 @@ def get_cli_auth_args(keystone_client): ) ) return " ".join(params) + + +def wait_for_url(url, ok_codes=None): + """Wait for url to return acceptable return code. + + :param url: url to test + :type url: str + :param ok_codes: HTTP codes that are acceptable + :type ok_codes: Optional[List[int]] + :raises: AssertionError + """ + if not ok_codes: + ok_codes = [requests.codes.ok] + for attempt in tenacity.Retrying( + stop=tenacity.stop_after_attempt(10), + wait=tenacity.wait_exponential( + multiplier=1, min=2, max=60)): + with attempt: + r = requests.get(url) + logging.info("{} returned {}".format(url, r.status_code)) + assert r.status_code in ok_codes + + +def get_keystone_overcloud_session_client(): + """Return keystone client for overcloud. + + If keystone is still in a transient state then it may take a few retries + before a client is returned + """ + for attempt in tenacity.Retrying( + stop=tenacity.stop_after_attempt(10), + wait=tenacity.wait_exponential( + multiplier=1, min=2, max=60)): + with attempt: + overcloud_auth = get_overcloud_auth() + wait_for_url(overcloud_auth['OS_AUTH_URL']) + session = get_overcloud_keystone_session() + keystone_client = get_keystone_session_client(session) + + return keystone_client