Made initial changes. #CALM-39459 (nutanix#300)

Environment decoupling support in DSL.
Added provider compilation templates for ntnx, vmware, gcp, aws.
Use command: calm decompile environment  -n <env_name> -p <project_name>

---------

Co-authored-by: Abhijeet Kaurav <[email protected]>
Co-authored-by: Utkarsh Bairolia <[email protected]>
(cherry picked from commit 70896ec9005e3ee5c2a4a5fd7c9d824c1c1c85ab)

calm/dsl/builtins/models/providers.py

@@ -11,6 +11,22 @@ class AccountProviderType(EntityType):
     __schema_name__ = "AccountProvider"
     __openapi_type__ = "app_account_provider"
 
+    @classmethod
+    def pre_decompile(mcls, cdict, context=[], prefix=""):
+
+        for _i in cdict.get("subnet_references", []):
+            _i["kind"] = "subnet"
+        cdict["subnet_reference_list"] = cdict.pop("subnet_references", [])
+
+        for _i in cdict.get("cluster_references", []):
+            _i["kind"] = "cluster"
+        cdict["cluster_reference_list"] = cdict.pop("cluster_references", [])
+
+        if cdict.get("default_subnet_reference", {}):
+            cdict["default_subnet_reference"]["kind"] = "subnet"
+
+        return cdict
+
 
 class AccountProviderValidator(PropertyValidator, openapi_type="app_account_provider"):
     __default__ = None

calm/dsl/cli/environment_commands.py

@@ -1,12 +1,13 @@
 import click
 
-from .main import get, delete, create, update, compile
+from .main import get, delete, create, update, compile, decompile
 from .environments import (
     create_environment_from_dsl_file,
     get_environment_list,
     delete_environment,
     update_environment_from_dsl_file,
     compile_environment_command,
+    decompile_environment_command,
 )
 
 from calm.dsl.log import get_logging_handle
@@ -156,3 +157,34 @@ def _compile_environment_command(env_file, project_name, out):
     """Compiles a DSL (Python) environment into JSON or YAML"""
 
     compile_environment_command(env_file, project_name, out)
+
+
+@decompile.command("environment", experimental=True)
+@click.option(
+    "--name",
+    "-n",
+    "name",
+    default=None,
+    help="Environment name",
+)
+@click.option(
+    "--file",
+    "-f",
+    "environment_file",
+    type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True),
+    help="Path to Environment file",
+)
+@click.option("--project", "-p", "project_name", help="Project name")
+@click.option(
+    "--dir",
+    "-d",
+    "environment_dir",
+    default=None,
+    help="Environment directory location used for placing decompiled entities",
+)
+def _decompile_environment_command(
+    name, environment_file, project_name, environment_dir
+):
+    """Decompiles environment present on server or json file"""
+
+    decompile_environment_command(name, environment_file, project_name, environment_dir)

calm/dsl/cli/environments.py

@@ -2,19 +2,30 @@
 import uuid
 import click
 import json
+import os
 import time
 import arrow
 from prettytable import PrettyTable
 from ruamel import yaml
 
 from calm.dsl.config import get_context
 from calm.dsl.api import get_api_client
-from calm.dsl.builtins import create_environment_payload, Environment
+from calm.dsl.builtins import (
+    create_environment_payload,
+    Environment,
+    get_valid_identifier,
+    MetadataType,
+    CredentialType,
+)
 from calm.dsl.builtins.models.helper.common import get_project
+from calm.dsl.decompile.main import init_decompile_context
+from calm.dsl.decompile.decompile_render import create_environment_dir
+from calm.dsl.decompile.file_handler import get_environment_dir
 from calm.dsl.tools import get_module_from_file
 from calm.dsl.store import Cache
 from calm.dsl.constants import CACHE
 from calm.dsl.log import get_logging_handle
+from calm.dsl.builtins.models.environment import EnvironmentType
 
 from .utils import (
     get_name_query,
@@ -551,6 +562,96 @@ def delete_environment(environment_name, project_name, no_cache_update=False):
         LOG.info("[Done]")
 
 
+def decompile_environment_command(
+    name, environment_file, project, environment_dir=None
+):
+    """helper to decompile environment"""
+    if name and environment_file:
+        LOG.error(
+            "Please provide either environment file location or server environment name"
+        )
+        sys.exit("Both environment name and file location provided.")
+    init_decompile_context()
+
+    if name:
+        decompile_environment_from_server(
+            name=name, environment_dir=environment_dir, project=project
+        )
+
+    elif environment_file:
+        decompile_environment_from_file(
+            filename=environment_file, environment_dir=environment_dir
+        )
+    else:
+        LOG.error(
+            "Please provide either environment file location or server environment name"
+        )
+        sys.exit("Environment name or file location not provided.")
+
+
+def decompile_environment_from_server(name, environment_dir, project):
+    """decompiles the environment by fetching it from server"""
+
+    client = get_api_client()
+    environment = get_environment(name, project)
+    environment_uuid = environment["status"]["uuid"]
+    res, err = client.environment.read(environment_uuid)
+    if err:
+        LOG.error(err)
+        sys.exit("Not able to decompile environment from server.")
+
+    environment = res.json()
+    _decompile_environment(
+        environment_payload=environment, environment_dir=environment_dir
+    )
+
+
+def decompile_environment_from_file(filename, environment_dir):
+    """decompile environment from local environment file"""
+
+    environment_payload = json.loads(open(filename).read())
+    _decompile_environment(
+        environment_payload=environment_payload, environment_dir=environment_dir
+    )
+
+
+def _decompile_environment(environment_payload, environment_dir):
+    """decompiles the environment from payload"""
+
+    environment_name = environment_payload["status"].get("name", "DslEnvironment")
+    environment_description = environment_payload["status"].get("description", "")
+
+    environment_metadata = environment_payload["metadata"]
+    # POP unnecessary keys
+    environment_metadata.pop("creation_time", None)
+    environment_metadata.pop("last_update_time", None)
+
+    metadata_obj = MetadataType.decompile(environment_metadata)
+
+    LOG.info("Decompiling environment {}".format(environment_name))
+    environment_cls = EnvironmentType.decompile(
+        environment_payload["status"]["resources"]
+    )
+
+    credentials = environment_cls.credentials
+
+    environment_cls.__name__ = get_valid_identifier(environment_name)
+    environment_cls.__doc__ = environment_description
+
+    create_environment_dir(
+        environment_cls=environment_cls,
+        environment_dir=environment_dir,
+        metadata_obj=metadata_obj,
+        credentials=credentials,
+    )
+    click.echo(
+        "\nSuccessfully decompiled. Directory location: {}. Environment location: {}".format(
+            highlight_text(get_environment_dir()),
+            highlight_text(os.path.join(get_environment_dir(), "environment.py")),
+        )
+    )
+
+
 def is_environment_exist(env_name, project_name):
     client = get_api_client()
     payload = {

calm/dsl/decompile/decompile_render.py

@@ -4,7 +4,12 @@
 from calm.dsl.log import get_logging_handle
 from calm.dsl.decompile.bp_file_helper import render_bp_file_template
 from calm.dsl.decompile.runbook import render_runbook_template
-from calm.dsl.decompile.file_handler import init_bp_dir, init_runbook_dir
+from calm.dsl.decompile.file_handler import (
+    init_bp_dir,
+    init_runbook_dir,
+    init_environment_dir,
+)
+from calm.dsl.decompile.environments import render_environment_template
 
 LOG = get_logging_handle(__name__)
 
@@ -23,6 +28,13 @@ def create_runbook_file(dir_name, runbook_data):
         fd.write(runbook_data)
 
 
+def create_environment_file(dir_name, environment_data):
+
+    environment_path = os.path.join(dir_name, "environment.py")
+    with open(environment_path, "w") as fd:
+        fd.write(environment_data)
+
+
 def create_bp_dir(
     bp_cls=None,
     bp_dir=None,
@@ -73,3 +85,27 @@ def create_runbook_dir(
     runbook_data = format_str(runbook_data, mode=FileMode())
     LOG.info("Creating runbook file")
     create_runbook_file(runbook_dir, runbook_data)
+
+
+def create_environment_dir(
+    environment_cls=None,
+    environment_dir=None,
+    metadata_obj=None,
+    credentials=None,
+):
+    if not environment_dir:
+        environment_dir = os.path.join(os.getcwd(), environment_cls.__name__)
+
+    LOG.info("Creating environment directory")
+    _, _, _, _ = init_environment_dir(environment_dir)
+    LOG.info("Rendering environment file template")
+    environment_data = render_environment_template(
+        environment_cls=environment_cls,
+        credentials=credentials,
+        metadata_obj=metadata_obj,
+    )
+
+    LOG.info("Formatting environment file using black")
+    environment_data = format_str(environment_data, mode=FileMode())
+    LOG.info("Creating environment file")
+    create_environment_file(environment_dir, environment_data)

calm/dsl/decompile/environments.py

@@ -0,0 +1,86 @@
+from calm.dsl.decompile.render import render_template
+from calm.dsl.decompile.ref import render_ref_template
+from calm.dsl.decompile.provider import render_provider_template
+from calm.dsl.decompile.substrate import render_substrate_template
+from calm.dsl.builtins.models.environment import EnvironmentType
+from .decompile_helpers import process_variable_name
+from calm.dsl.decompile.variable import get_secret_variable_files
+from calm.dsl.builtins import get_valid_identifier
+
+from calm.dsl.decompile.credential import (
+    render_credential_template,
+    get_cred_files,
+    get_cred_var_name,
+)
+
+from calm.dsl.log import get_logging_handle
+
+LOG = get_logging_handle(__name__)
+
+
+def render_environment_template(
+    environment_cls,
+    metadata_obj=None,
+    entity_context="",
+    CONFIG_SPEC_MAP={},
+    credentials=[],
+):
+    LOG.debug("Rendering {} environment template".format(environment_cls.__name__))
+    if not isinstance(environment_cls, EnvironmentType):
+        raise TypeError("{} is not of type {}".format(environment_cls, environment))
+
+    # Update entity context
+    entity_context = entity_context + "_Environment_" + environment_cls.__name__
+
+    environment_name = getattr(environment_cls, "name", "") or environment_cls.__name__
+
+    rendered_credential_list = []
+    credentials_list = []
+    for cred in credentials:
+        rendered_credential_list.append(render_credential_template(cred))
+        credentials_list.append(get_cred_var_name(cred.name))
+
+    # Getting the local files used for secrets
+    secret_files = get_secret_variable_files()
+    secret_files.extend(get_cred_files())
+
+    class_name = "ENV_{}".format(get_valid_identifier(environment_cls.__name__))
+
+    user_attrs = {
+        "name": class_name,
+        "credentials": rendered_credential_list,
+        "credentials_list": credentials_list,
+        "secret_files": secret_files,
+    }
+
+    rendered_substrates_list = []
+    substrates_list = []
+    substrate_name_counter = 1
+    if environment_cls.substrates:
+        for substrate in environment_cls.substrates:
+            if substrate.name in substrates_list:
+                new_name = "{}_{}".format(substrate.name, str(substrate_name_counter))
+                substrates_list.append(new_name)
+
+                substrate.__name__ = new_name
+                rendered_substrates_list.append(render_substrate_template(substrate))
+
+                substrate_name_counter += 1
+            else:
+                rendered_substrates_list.append(render_substrate_template(substrate))
+                substrates_list.append(substrate.name)
+    user_attrs["substrates"] = rendered_substrates_list
+    user_attrs["substrates_list"] = substrates_list
+
+    rendered_providers_list = []
+    if environment_cls.providers:
+        for provider in environment_cls.providers:
+            rendered_providers_list.append(render_provider_template(provider))
+    user_attrs["providers"] = rendered_providers_list
+
+    gui_display_name = getattr(environment_cls, "name", "") or environment_cls.__name__
+    if gui_display_name != environment_cls.__name__:
+        user_attrs["gui_display_name"] = gui_display_name
+
+    text = render_template(schema_file="environments.py.jinja2", obj=user_attrs)
+    return text.strip()

calm/dsl/decompile/file_handler.py

@@ -46,6 +46,26 @@ def make_runbook_dirs(runbook_dir):
     return (runbook_dir, local_dir, scripts_dir)
 
 
+def make_environment_dirs(environment_dir):
+
+    if not os.path.isdir(environment_dir):
+        os.makedirs(environment_dir)
+
+    local_dir = os.path.join(environment_dir, LOCAL_DIR_KEY)
+    if not os.path.isdir(local_dir):
+        os.makedirs(local_dir)
+
+    spec_dir = os.path.join(environment_dir, SPECS_DIR_KEY)
+    if not os.path.isdir(spec_dir):
+        os.makedirs(spec_dir)
+
+    scripts_dir = os.path.join(environment_dir, SCRIPTS_DIR_KEY)
+    if not os.path.isdir(scripts_dir):
+        os.makedirs(scripts_dir)
+
+    return (environment_dir, local_dir, spec_dir, scripts_dir)
+
+
 def init_bp_dir(bp_dir):
 
     global LOCAL_DIR, SCRIPTS_DIR, SPECS_DIR, BP_DIR
@@ -62,6 +82,16 @@ def init_runbook_dir(runbook_dir):
     return (RUNBOOK_DIR, LOCAL_DIR, SCRIPTS_DIR)
 
 
+def init_environment_dir(environment_dir):
+
+    global LOCAL_DIR, SCRIPTS_DIR, SPECS_DIR, ENVIRONMENT_DIR
+    ENVIRONMENT_DIR, LOCAL_DIR, SPECS_DIR, SCRIPTS_DIR = make_environment_dirs(
+        environment_dir
+    )
+
+    return (ENVIRONMENT_DIR, LOCAL_DIR, SPECS_DIR, SCRIPTS_DIR)
+
+
 def get_bp_dir():
     return BP_DIR
 
@@ -70,6 +100,10 @@ def get_runbook_dir():
     return RUNBOOK_DIR
 
 
+def get_environment_dir():
+    return ENVIRONMENT_DIR
+
+
 def get_local_dir():
     return LOCAL_DIR