diff --git a/Tests/iaas/key-manager/check-for-key-manager.py b/Tests/iaas/key-manager/check-for-key-manager.py index 20f62165d..dae49acdd 100755 --- a/Tests/iaas/key-manager/check-for-key-manager.py +++ b/Tests/iaas/key-manager/check-for-key-manager.py @@ -127,15 +127,16 @@ def main(): # parse cloud name for lookup in clouds.yaml cloud = args.os_cloud or os.environ.get("OS_CLOUD", None) if not cloud: - raise RuntimeError( + logger.critical( "You need to have the OS_CLOUD environment variable set to your cloud " "name or pass it via --os-cloud" ) + return 2 with openstack.connect(cloud=cloud) as conn: if not check_for_member_role(conn): logger.critical("Cannot test key-manager permissions. User has wrong roles") - return 1 + return 2 if check_presence_of_key_manager(conn): return check_key_manager_permissions(conn) else: @@ -145,9 +146,11 @@ def main(): if __name__ == "__main__": try: - sys.exit(main()) - except SystemExit: + sys.exit(main() or 0) + except SystemExit as e: + if e.code < 2: + print("key-manager-check: " + ('PASS', 'FAIL')[min(1, e.code)]) raise except BaseException: logger.critical("exception", exc_info=True) - sys.exit(1) + sys.exit(2) diff --git a/Tests/iaas/security-groups/default-security-group-rules.py b/Tests/iaas/security-groups/default-security-group-rules.py index a71f333e8..def511956 100755 --- a/Tests/iaas/security-groups/default-security-group-rules.py +++ b/Tests/iaas/security-groups/default-security-group-rules.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 """Default Security Group Rules Checker This script tests the absence of any ingress default security group rule diff --git a/Tests/iaas/volume-backup/volume-backup-tester.py b/Tests/iaas/volume-backup/volume-backup-tester.py old mode 100644 new mode 100755 index 83ec56e8b..bcbb89664 --- a/Tests/iaas/volume-backup/volume-backup-tester.py +++ b/Tests/iaas/volume-backup/volume-backup-tester.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 """Volume Backup API tester for Block Storage API This test script executes basic operations on the Block Storage API centered @@ -14,10 +15,11 @@ import argparse import getpass +import logging import os +import sys import time import typing -import logging import openstack @@ -30,43 +32,23 @@ WAIT_TIMEOUT = 60 -class ConformanceTestException(Exception): - pass - - -def ensure(condition: bool, error_message: str): - """ - Custom replacement for the `assert` statement that is not removed by the - -O optimization parameter. - If the condition does not evaluate to `True`, a ConformanceTestException - will be raised containing the specified error_message string. - """ - if not condition: - raise ConformanceTestException(error_message) - - -def connect(cloud_name: str, password: typing.Optional[str] = None - ) -> openstack.connection.Connection: - """Create a connection to an OpenStack cloud - - :param string cloud_name: - The name of the configuration to load from clouds.yaml. - - :param string password: - Optional password override for the connection. - - :returns: openstack.connnection.Connection - """ - - if password: - return openstack.connect( - cloud=cloud_name, - password=password - ) - else: - return openstack.connect( - cloud=cloud_name, - ) +def wait_for_resource( + get_func: typing.Callable[[str], openstack.resource.Resource], + resource_id: str, + expected_status=("available", ), + timeout=WAIT_TIMEOUT, +) -> None: + seconds_waited = 0 + resource = get_func(resource_id) + while resource is None or resource.status not in expected_status: + time.sleep(1.0) + seconds_waited += 1 + if seconds_waited >= timeout: + raise RuntimeError( + f"Timed out after {seconds_waited} s: waiting for resource {resource_id} " + f"to be in status {expected_status} (current: {resource and resource.status})" + ) + resource = get_func(resource_id) def test_backup(conn: openstack.connection.Connection, @@ -82,102 +64,50 @@ def test_backup(conn: openstack.connection.Connection, # CREATE VOLUME volume_name = f"{prefix}volume" logging.info(f"Creating volume '{volume_name}' ...") - volume = conn.block_storage.create_volume( - name=volume_name, - size=1 - ) - ensure( - volume is not None, - f"Creation of initial volume '{volume_name}' failed" - ) + volume = conn.block_storage.create_volume(name=volume_name, size=1) + if volume is None: + raise RuntimeError(f"Creation of initial volume '{volume_name}' failed") volume_id = volume.id - ensure( - conn.block_storage.get_volume(volume_id) is not None, - f"Retrieving initial volume by ID '{volume_id}' failed" - ) + if conn.block_storage.get_volume(volume_id) is None: + raise RuntimeError(f"Retrieving initial volume by ID '{volume_id}' failed") logging.info( f"↳ waiting for volume with ID '{volume_id}' to reach status " f"'available' ..." ) - seconds_waited = 0 - while conn.block_storage.get_volume(volume_id).status != "available": - time.sleep(1.0) - seconds_waited += 1 - ensure( - seconds_waited < timeout, - f"Timeout reached while waiting for volume to reach status " - f"'available' (volume id: {volume_id}) after {seconds_waited} " - f"seconds" - ) + wait_for_resource(conn.block_storage.get_volume, volume_id, timeout=timeout) logging.info("Create empty volume: PASS") # CREATE BACKUP logging.info("Creating backup from volume ...") - backup = conn.block_storage.create_backup( - name=f"{prefix}volume-backup", - volume_id=volume_id - ) - ensure( - backup is not None, - "Backup creation failed" - ) + backup = conn.block_storage.create_backup(name=f"{prefix}volume-backup", volume_id=volume_id) + if backup is None: + raise RuntimeError("Backup creation failed") backup_id = backup.id - ensure( - conn.block_storage.get_backup(backup_id) is not None, - "Retrieving backup by ID failed" - ) + if conn.block_storage.get_backup(backup_id) is None: + raise RuntimeError("Retrieving backup by ID failed") logging.info(f"↳ waiting for backup '{backup_id}' to become available ...") - seconds_waited = 0 - while conn.block_storage.get_backup(backup_id).status != "available": - time.sleep(1.0) - seconds_waited += 1 - ensure( - seconds_waited < timeout, - f"Timeout reached while waiting for backup to reach status " - f"'available' (backup id: {backup_id}) after {seconds_waited} " - f"seconds" - ) + wait_for_resource(conn.block_storage.get_backup, backup_id, timeout=timeout) logging.info("Create backup from volume: PASS") # RESTORE BACKUP restored_volume_name = f"{prefix}restored-backup" logging.info(f"Restoring backup to volume '{restored_volume_name}' ...") - conn.block_storage.restore_backup( - backup_id, - name=restored_volume_name - ) + conn.block_storage.restore_backup(backup_id, name=restored_volume_name) logging.info( f"↳ waiting for restoration target volume '{restored_volume_name}' " f"to be created ..." ) - seconds_waited = 0 - while conn.block_storage.find_volume(restored_volume_name) is None: - time.sleep(1.0) - seconds_waited += 1 - ensure( - seconds_waited < timeout, - f"Timeout reached while waiting for restored volume to be created " - f"(volume name: {restored_volume_name}) after {seconds_waited} " - f"seconds" - ) + wait_for_resource(conn.block_storage.find_volume, restored_volume_name, timeout=timeout) # wait for the volume restoration to finish logging.info( f"↳ waiting for restoration target volume '{restored_volume_name}' " f"to reach 'available' status ..." ) volume_id = conn.block_storage.find_volume(restored_volume_name).id - while conn.block_storage.get_volume(volume_id).status != "available": - time.sleep(1.0) - seconds_waited += 1 - ensure( - seconds_waited < timeout, - f"Timeout reached while waiting for restored volume reach status " - f"'available' (volume id: {volume_id}) after {seconds_waited} " - f"seconds" - ) + wait_for_resource(conn.block_storage.get_volume, volume_id, timeout=timeout) logging.info("Restore volume from backup: PASS") @@ -190,54 +120,32 @@ def cleanup(conn: openstack.connection.Connection, prefix=DEFAULT_PREFIX, resources behind. Otherwise returns True to indicate cleanup success. """ - def wait_for_resource(resource_type: str, resource_id: str, - expected_status=("available", )) -> None: - seconds_waited = 0 - get_func = getattr(conn.block_storage, f"get_{resource_type}") - while get_func(resource_id).status not in expected_status: - time.sleep(1.0) - seconds_waited += 1 - ensure( - seconds_waited < timeout, - f"Timeout reached while waiting for {resource_type} during " - f"cleanup to be in status {expected_status} " - f"({resource_type} id: {resource_id}) after {seconds_waited} " - f"seconds" - ) - - logging.info(f"Performing cleanup for resources with the " - f"'{prefix}' prefix ...") + logging.info(f"Performing cleanup for resources with the '{prefix}' prefix ...") - cleanup_was_successful = True + cleanup_issues = 0 # count failed cleanup operations backups = conn.block_storage.backups() for backup in backups: - if backup.name.startswith(prefix): - try: - wait_for_resource( - "backup", backup.id, - expected_status=("available", "error") - ) - except openstack.exceptions.ResourceNotFound: - # if the resource has vanished on - # its own in the meantime ignore it - continue - except ConformanceTestException as e: - # This exception happens if the backup state does not reach any - # of the desired ones specified above. We do not need to set - # cleanup_was_successful to False here since any remaining ones - # will be caught in the next loop down below anyway. - logging.warning(str(e)) - else: - logging.info(f"↳ deleting volume backup '{backup.id}' ...") - # Setting ignore_missing to False here will make an exception - # bubble up in case the cinder-backup service is not present. - # Since we already catch ResourceNotFound for the backup above, - # the absence of the cinder-backup service is the only - # NotFoundException that is left to be thrown here. - # We treat this as a fatal due to the cinder-backup service - # being mandatory. - conn.block_storage.delete_backup( - backup.id, ignore_missing=False) + if not backup.name.startswith(prefix): + continue + try: + # we can only delete if status is available or error, so try and wait + wait_for_resource( + conn.block_storage.get_backup, + backup.id, + expected_status=("available", "error"), + timeout=timeout, + ) + logging.info(f"↳ deleting volume backup '{backup.id}' ...") + conn.block_storage.delete_backup(backup.id) + except openstack.exceptions.ResourceNotFound: + # if the resource has vanished on its own in the meantime ignore it + continue + except Exception as e: + # Most common exception would be a timeout in wait_for_resource. + # We do not need to increment cleanup_issues here since + # any remaining ones will be caught in the next loop down below anyway. + logging.debug("traceback", exc_info=True) + logging.warning(str(e)) # wait for all backups to be cleaned up before attempting to remove volumes seconds_waited = 0 @@ -248,7 +156,7 @@ def wait_for_resource(resource_type: str, resource_id: str, time.sleep(1.0) seconds_waited += 1 if seconds_waited >= timeout: - cleanup_was_successful = False + cleanup_issues += 1 logging.warning( f"Timeout reached while waiting for all backups with prefix " f"'{prefix}' to finish deletion during cleanup after " @@ -258,21 +166,31 @@ def wait_for_resource(resource_type: str, resource_id: str, volumes = conn.block_storage.volumes() for volume in volumes: - if volume.name.startswith(prefix): - try: - wait_for_resource("volume", volume.id, expected_status=("available", "error")) - except openstack.exceptions.ResourceNotFound: - # if the resource has vanished on - # its own in the meantime ignore it - continue - except ConformanceTestException as e: - logging.warning(str(e)) - cleanup_was_successful = False - else: - logging.info(f"↳ deleting volume '{volume.id}' ...") - conn.block_storage.delete_volume(volume.id) + if not volume.name.startswith(prefix): + continue + try: + wait_for_resource( + conn.block_storage.get_volume, + volume.id, + expected_status=("available", "error"), + timeout=timeout, + ) + logging.info(f"↳ deleting volume '{volume.id}' ...") + conn.block_storage.delete_volume(volume.id) + except openstack.exceptions.ResourceNotFound: + # if the resource has vanished on its own in the meantime ignore it + continue + except Exception as e: + logging.debug("traceback", exc_info=True) + logging.warning(str(e)) + cleanup_issues += 1 + + if cleanup_issues: + logging.info( + f"Some resources with the '{prefix}' prefix were not cleaned up!" + ) - return cleanup_was_successful + return not cleanup_issues def main(): @@ -320,35 +238,37 @@ def main(): ) # parse cloud name for lookup in clouds.yaml - cloud = os.environ.get("OS_CLOUD", None) - if args.os_cloud: - cloud = args.os_cloud + cloud = args.os_cloud or os.environ.get("OS_CLOUD", None) if not cloud: raise Exception( "You need to have the OS_CLOUD environment variable set to your " "cloud name or pass it via --os-cloud" ) - conn = connect( - cloud, - password=getpass.getpass("Enter password: ") if args.ask else None - ) + password = getpass.getpass("Enter password: ") if args.ask else None - if not cleanup(conn, prefix=args.prefix, timeout=args.timeout): - raise Exception( - f"Cleanup was not successful, there may be leftover resources " - f"with the '{args.prefix}' prefix" - ) - if args.cleanup_only: - return - try: - test_backup(conn, prefix=args.prefix, timeout=args.timeout) - finally: + with openstack.connect(cloud, password=password) as conn: if not cleanup(conn, prefix=args.prefix, timeout=args.timeout): - logging.info( - f"There may be leftover resources with the " - f"'{args.prefix}' prefix that could not be cleaned up!" - ) + raise RuntimeError("Initial cleanup failed") + if args.cleanup_only: + logging.info("Cleanup-only run finished.") + return + try: + test_backup(conn, prefix=args.prefix, timeout=args.timeout) + except BaseException: + print('volume-backup-check: FAIL') + raise + else: + print('volume-backup-check: PASS') + finally: + cleanup(conn, prefix=args.prefix, timeout=args.timeout) if __name__ == "__main__": - main() + try: + sys.exit(main()) + except SystemExit: + raise + except BaseException as exc: + logging.debug("traceback", exc_info=True) + logging.critical(str(exc)) + sys.exit(1) diff --git a/Tests/iaas/volume-types/volume-types-check.py b/Tests/iaas/volume-types/volume-types-check.py old mode 100644 new mode 100755 index 444755816..4b1945fb8 --- a/Tests/iaas/volume-types/volume-types-check.py +++ b/Tests/iaas/volume-types/volume-types-check.py @@ -141,6 +141,8 @@ def main(argv): "Total critical / error / warning: " f"{c[logging.CRITICAL]} / {c[logging.ERROR]} / {c[logging.WARNING]}" ) + if not c[logging.CRITICAL]: + print("volume-types-check: " + ('PASS', 'FAIL')[min(1, c[logging.ERROR])]) return min(127, c[logging.CRITICAL] + c[logging.ERROR]) # cap at 127 due to OS restrictions diff --git a/Tests/iam/domain-manager/domain-manager-check.py b/Tests/iam/domain-manager/domain-manager-check.py old mode 100644 new mode 100755 index e56aad884..41040122b --- a/Tests/iam/domain-manager/domain-manager-check.py +++ b/Tests/iam/domain-manager/domain-manager-check.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 """Domain Manager policy configuration checker This script uses the OpenStack SDK to validate the proper implementation diff --git a/Tests/scs-compatible-iaas.yaml b/Tests/scs-compatible-iaas.yaml index 0d9c0ee61..5ad119fbf 100644 --- a/Tests/scs-compatible-iaas.yaml +++ b/Tests/scs-compatible-iaas.yaml @@ -154,7 +154,75 @@ modules: tags: [mandatory] description: > Must fulfill all requirements of + - id: scs-0114-v1 + name: Volume Types + url: https://docs.scs.community/standards/scs-0114-v1-volume-type-standard + run: + - executable: ./iaas/volume-types/volume-types-check.py + args: -c {os_cloud} -d + testcases: + - id: volume-types-check + tags: [mandatory] + description: > + Must fulfill all requirements of + - id: scs-0115-v1 + name: Default rules for security groups + url: https://docs.scs.community/standards/scs-0115-v1-default-rules-for-security-groups + run: + - executable: ./iaas/security-groups/default-security-group-rules.py + args: --os-cloud {os_cloud} --debug + testcases: + - id: security-groups-default-rules-check + tags: [mandatory] + description: > + Must fulfill all requirements of + - id: scs-0116-v1 + name: Key manager + url: https://docs.scs.community/standards/scs-0116-v1-key-manager-standard + run: + - executable: ./iaas/key-manager/check-for-key-manager.py + args: --os-cloud {os_cloud} --debug + testcases: + - id: key-manager-check + tags: [mandatory] + description: > + Must fulfill all requirements of + - id: scs-0117-v1 + name: Volume backup + url: https://docs.scs.community/standards/scs-0117-v1-volume-backup-service + run: + - executable: ./iaas/volume-backup/volume-backup-tester.py + args: --os-cloud {os_cloud} --debug + testcases: + - id: volume-backup-check + tags: [mandatory] + description: > + Must fulfill all requirements of + - id: scs-0121-v1 + name: Availability Zones + url: https://docs.scs.community/standards/scs-0121-v1-Availability-Zones-Standard + testcases: + - id: availability-zones-check + tags: [availability-zones] + description: > + Note: manual check! Must fulfill all requirements of + - id: scs-0302-v1 + name: Domain Manager Role + url: https://docs.scs.community/standards/scs-0302-v1-domain-manager-role + # run: + # - executable: ./iam/domain-manager/domain-manager-check.py + # args: --os-cloud {os_cloud} --debug --domain-config ... + testcases: + - id: domain-manager-check + tags: [domain-manager] + description: > + Note: manual check! Must fulfill all requirements of timeline: + - date: 2024-11-08 + versions: + v5: draft + v4: effective + v3: deprecated - date: 2024-08-23 versions: v5: draft @@ -202,8 +270,15 @@ versions: - ref: scs-0104-v1 parameters: image_spec: https://raw.githubusercontent.com/SovereignCloudStack/standards/main/Tests/iaas/scs-0104-v1-images-v5.yaml + - scs-0114-v1 + - scs-0115-v1 + - scs-0116-v1 + - scs-0117-v1 + - scs-0121-v1 + - scs-0302-v1 targets: main: mandatory + preview: domain-manager/availability-zones - version: v4 stabilized_at: 2024-02-28 include: