Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

support content path update #4663

Draft
wants to merge 46 commits into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
46 commits
Select commit Hold shift + click to select a range
e2e7d9a
support content path update
barryyosi-panw Nov 12, 2024
14a911d
changelog
barryyosi-panw Nov 12, 2024
84c0ed0
pre-commit fixes
barryyosi-panw Nov 12, 2024
f8d062f
docstring
barryyosi-panw Nov 12, 2024
54721cf
pre commit fix
barryyosi-panw Nov 12, 2024
b6b268d
introduce ContentPaths and update references
barryyosi-panw Nov 12, 2024
be4f402
complete content paths replacement
barryyosi-panw Nov 12, 2024
dee9a27
fixes
barryyosi-panw Nov 12, 2024
18df180
pre-commit fixes
barryyosi-panw Nov 12, 2024
7fb99a2
fix tests
barryyosi-panw Nov 12, 2024
61deb36
Merge branch 'master' into support-content-path-dynamic-update
barryyosi-panw Nov 13, 2024
0f391e0
unittests fix
barryyosi-panw Nov 13, 2024
869d9e8
unittests fix
barryyosi-panw Nov 13, 2024
5367198
tests fixes
barryyosi-panw Nov 13, 2024
68cbd96
test fixes
barryyosi-panw Nov 13, 2024
baf7eb5
test fixes
barryyosi-panw Nov 14, 2024
3fa8e86
Introducing 'DEMISTO_SDK_SKIP_LOGGER_SETUP' envvar.
barryyosi-panw Nov 14, 2024
6649827
ruff
barryyosi-panw Nov 14, 2024
35eb0ce
logger setup envvar resolution
barryyosi-panw Nov 14, 2024
157040f
ruff
barryyosi-panw Nov 14, 2024
2b1c502
set CONTENT_PATH within graph creation
barryyosi-panw Nov 14, 2024
97819c9
update content path in graph creation
barryyosi-panw Nov 14, 2024
5659403
update content path in graph creation
barryyosi-panw Nov 14, 2024
7d61313
mypy fix
barryyosi-panw Nov 14, 2024
11ab75e
content paths fix
barryyosi-panw Nov 14, 2024
511f3c7
revert
barryyosi-panw Nov 14, 2024
f222dc2
precommit
barryyosi-panw Nov 14, 2024
2978aff
fixes
barryyosi-panw Nov 14, 2024
77800e7
fix graph tests
barryyosi-panw Nov 18, 2024
5e3e6d5
fix graph tests
barryyosi-panw Nov 18, 2024
4d6fd25
fix graph tests
barryyosi-panw Nov 18, 2024
b7dc0de
fix graph tests
barryyosi-panw Nov 18, 2024
521b4b0
fix graph tests
barryyosi-panw Nov 18, 2024
c670acf
fix graph tests
barryyosi-panw Nov 18, 2024
4b3b257
fix graph tests
barryyosi-panw Nov 18, 2024
c33d656
fix graph tests
barryyosi-panw Nov 18, 2024
181c2bf
fix graph tests
barryyosi-panw Nov 18, 2024
fa4f18f
fix graph tests
barryyosi-panw Nov 18, 2024
29043d6
fix graph tests
barryyosi-panw Nov 18, 2024
64745f1
fix graph tests
barryyosi-panw Nov 18, 2024
5e75193
fix graph tests
barryyosi-panw Nov 18, 2024
81f13b7
fix graph tests
barryyosi-panw Nov 19, 2024
a31a033
Merge branch 'master' into support-content-path-dynamic-update
barryyosi-panw Nov 19, 2024
e153099
fix graph tests
barryyosi-panw Nov 19, 2024
fa41bd2
fix integration tests
barryyosi-panw Nov 19, 2024
d95d5a0
fix unit tests
barryyosi-panw Nov 19, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .changelog/4663.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
changes:
- description: Introducing functionality for updating CONTENT_PATH global.
type: feature
pr_number: 4663
2 changes: 2 additions & 0 deletions TestSuite/repo.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
from typing import List, Optional

from demisto_sdk.commands.common.constants import DEMISTO_GIT_PRIMARY_BRANCH
from demisto_sdk.commands.common.content_constant_paths import ContentPaths
from demisto_sdk.commands.common.git_util import GitUtil
from demisto_sdk.commands.content_graph.commands.create import create_content_graph
from demisto_sdk.commands.content_graph.interface import (
Expand Down Expand Up @@ -43,6 +44,7 @@ def __init__(self, tmpdir: Path, init_git: bool = False):
self._packs_path: Path = tmpdir / "Packs"
self._packs_path.mkdir()
self.path = str(self._tmpdir)
ContentPaths.update_content_path(self.path)

# Initiate ./Tests/ dir
self._test_dir = tmpdir / "Tests"
Expand Down
17 changes: 8 additions & 9 deletions conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@

import os
import shutil
from pathlib import Path
from typing import Generator
from unittest import mock

Expand All @@ -24,13 +23,15 @@


def get_repo(request: FixtureRequest, tmp_path_factory: TempPathFactory) -> Repo:
tmp_dir = _mk_tmp(request, tmp_path_factory)
return Repo(tmp_dir)
content_tmp_dir = _mk_tmp(request, tmp_path_factory) / "content"
content_tmp_dir.mkdir()
return Repo(content_tmp_dir)


def get_git_repo(request: FixtureRequest, tmp_path_factory: TempPathFactory) -> Repo:
tmp_dir = _mk_tmp(request, tmp_path_factory)
return Repo(tmp_dir, init_git=True)
content_tmp_dir = _mk_tmp(request, tmp_path_factory) / "content"
content_tmp_dir.mkdir()
return Repo(content_tmp_dir, init_git=True)


def get_pack(request: FixtureRequest, tmp_path_factory: TempPathFactory) -> Pack:
Expand Down Expand Up @@ -101,11 +102,9 @@ def graph_repo(request: FixtureRequest, tmp_path_factory: TempPathFactory) -> Ge
import demisto_sdk.commands.content_graph.objects.base_content as bc

repo = get_repo(request, tmp_path_factory)
neo4j_path = bc.ContentPaths.CONTENT_PATH.parent.parent / "neo4j"

bc.CONTENT_PATH = Path(repo.path)
neo4j_path = bc.CONTENT_PATH.parent.parent / "neo4j"

mock.patch.object(ContentGraphInterface, "repo_path", bc.CONTENT_PATH)
mock.patch.object(ContentGraphInterface, "repo_path", bc.ContentPaths.CONTENT_PATH)
mock.patch.object(neo4j_service, "REPO_PATH", neo4j_path)

yield repo
Expand Down
8 changes: 7 additions & 1 deletion demisto_sdk/__init__.py
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This fixes the change introduced here #4657

Original file line number Diff line number Diff line change
@@ -1,4 +1,10 @@
if __name__ in ["__main__", "demisto_sdk"]:
import os

if os.environ.get("DEMISTO_SDK_SKIP_LOGGER_SETUP", "False").lower() not in [
"true",
"yes",
"1",
]:
from demisto_sdk.commands.common.logger import logging_setup

logging_setup(initial=True, calling_function="__init__")
17 changes: 8 additions & 9 deletions demisto_sdk/__main__.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,10 +37,7 @@
FileType,
MarketplaceVersions,
)
from demisto_sdk.commands.common.content_constant_paths import (
ALL_PACKS_DEPENDENCIES_DEFAULT_PATH,
CONTENT_PATH,
)
from demisto_sdk.commands.common.content_constant_paths import ContentPaths
from demisto_sdk.commands.common.cpu_count import cpu_count
from demisto_sdk.commands.common.handlers import DEFAULT_JSON_HANDLER as json
from demisto_sdk.commands.common.hook_validations.readme import ReadMeValidator
Expand Down Expand Up @@ -212,7 +209,7 @@ def main(ctx, config, version, release_notes, **kwargs):
import dotenv

dotenv.load_dotenv(
CONTENT_PATH / ".env", override=True
ContentPaths.CONTENT_PATH / ".env", override=True
) # load .env file from the cwd

if platform.system() == "Windows":
Expand Down Expand Up @@ -2712,7 +2709,9 @@ def find_dependencies(ctx, **kwargs):
use_pack_metadata = kwargs.get("use_pack_metadata", False)
all_packs_dependencies = kwargs.get("all_packs_dependencies", False)
get_dependent_on = kwargs.get("get_dependent_on", False)
output_path = kwargs.get("output_path", ALL_PACKS_DEPENDENCIES_DEFAULT_PATH)
output_path = kwargs.get(
"output_path", ContentPaths.ALL_PACKS_DEPENDENCIES_DEFAULT_PATH
)
dependency = kwargs.get("dependency", "")
try:
PackDependencies.find_dependencies_manager(
Expand Down Expand Up @@ -3644,12 +3643,12 @@ def setup_env(

if ide == "auto-detect":
# Order decides which IDEType will be selected for configuration if multiple IDEs are detected
if (CONTENT_PATH / ".vscode").exists():
if (ContentPaths.CONTENT_PATH / ".vscode").exists():
logger.info(
"Visual Studio Code IDEType has been detected and will be configured."
)
ide_type = IDEType.VSCODE
elif (CONTENT_PATH / ".idea").exists():
elif (ContentPaths.CONTENT_PATH / ".idea").exists():
logger.info(
"PyCharm / IDEA IDEType has been detected and will be configured."
)
Expand Down Expand Up @@ -3912,7 +3911,7 @@ def xsoar_linter(
def dump_api(
ctx: typer.Context,
output_path: Path = typer.Option(
CONTENT_PATH,
ContentPaths.CONTENT_PATH,
"-o",
"--output",
help="The output directory or JSON file to save the demisto-sdk api.",
Expand Down
114 changes: 79 additions & 35 deletions demisto_sdk/commands/common/content_constant_paths.py
Original file line number Diff line number Diff line change
@@ -1,45 +1,89 @@
from pathlib import Path
from typing import Union

from demisto_sdk.commands.common.constants import NATIVE_IMAGE_FILE_NAME, TESTS_DIR
from demisto_sdk.commands.common.logger import logger
from demisto_sdk.commands.common.tools import get_content_path

CONTENT_PATH: Path = Path(get_content_path())

ALL_PACKS_DEPENDENCIES_DEFAULT_PATH = CONTENT_PATH / "all_packs_dependencies.json"

CONF_PATH = CONTENT_PATH / TESTS_DIR / "conf.json"

DEFAULT_ID_SET_PATH = CONTENT_PATH / TESTS_DIR / "id_set.json"
MP_V2_ID_SET_PATH = CONTENT_PATH / TESTS_DIR / "id_set_mp_v2.json"
XPANSE_ID_SET_PATH = CONTENT_PATH / TESTS_DIR / "id_set_xpanse.json"
LANDING_PAGE_SECTIONS_PATH = (
CONTENT_PATH / TESTS_DIR / "Marketplace" / "landingPage_sections.json"
)
NATIVE_IMAGE_PATH = CONTENT_PATH / "Tests" / NATIVE_IMAGE_FILE_NAME

COMMON_SERVER_PYTHON_PATH = (
CONTENT_PATH / "Packs" / "Base" / "Scripts" / "CommonServerPython"
)
DEMISTO_MOCK_PATH = CONTENT_PATH / TESTS_DIR / "demistomock"
API_MODULES_SCRIPTS_DIR = CONTENT_PATH / "Packs" / "ApiModules" / "Scripts"

PYTHONPATH = [
path.absolute()
for path in [
Path(CONTENT_PATH),
COMMON_SERVER_PYTHON_PATH,
DEMISTO_MOCK_PATH,
Path(__file__).parent.parent / "lint" / "resources" / "pylint_plugins",

class ContentPaths:
# Class-level variables (shared across all instances)
CONTENT_PATH: Path = Path(get_content_path())
ALL_PACKS_DEPENDENCIES_DEFAULT_PATH: Path = (
CONTENT_PATH / "all_packs_dependencies.json"
)
CONF_PATH: Path = CONTENT_PATH / TESTS_DIR / "conf.json"
DEFAULT_ID_SET_PATH: Path = CONTENT_PATH / TESTS_DIR / "id_set.json"
MP_V2_ID_SET_PATH: Path = CONTENT_PATH / TESTS_DIR / "id_set_mp_v2.json"
XPANSE_ID_SET_PATH: Path = CONTENT_PATH / TESTS_DIR / "id_set_xpanse.json"
LANDING_PAGE_SECTIONS_PATH: Path = (
CONTENT_PATH / TESTS_DIR / "Marketplace" / "landingPage_sections.json"
)
NATIVE_IMAGE_PATH: Path = CONTENT_PATH / TESTS_DIR / NATIVE_IMAGE_FILE_NAME
COMMON_SERVER_PYTHON_PATH: Path = (
CONTENT_PATH / "Packs" / "Base" / "Scripts" / "CommonServerPython"
)
DEMISTO_MOCK_PATH: Path = CONTENT_PATH / TESTS_DIR / "demistomock"
API_MODULES_SCRIPTS_DIR: Path = CONTENT_PATH / "Packs" / "ApiModules" / "Scripts"
PYTHONPATH = [
path.absolute()
for path in [
CONTENT_PATH,
COMMON_SERVER_PYTHON_PATH,
DEMISTO_MOCK_PATH,
Path(__file__).parent.parent / "lint" / "resources" / "pylint_plugins",
]
]
]
PYTHONPATH_STR = ":".join(str(path) for path in PYTHONPATH)

if API_MODULES_SCRIPTS_DIR.exists():
PYTHONPATH.extend(path.absolute() for path in API_MODULES_SCRIPTS_DIR.iterdir())
@classmethod
def update_content_path(cls, content_path: Union[str, Path]) -> None:
"""
Updates the class-level content path and derived paths.

else:
logger.debug(
"Could not add API modules to 'PYTHONPATH' as the base directory does not exist."
)
Args:
content_path (Union[str, Path]): The new content path to set.
"""
logger.info(f"Updating content_path globally: {content_path}")

cls.CONTENT_PATH = Path(content_path)
logger.info(f"CONTENT_PATH Type: {type(cls.CONTENT_PATH)}")
cls.ALL_PACKS_DEPENDENCIES_DEFAULT_PATH = (
cls.CONTENT_PATH / "all_packs_dependencies.json"
)
cls.CONF_PATH = cls.CONTENT_PATH / TESTS_DIR / "conf.json"
cls.DEFAULT_ID_SET_PATH = cls.CONTENT_PATH / TESTS_DIR / "id_set.json"
cls.MP_V2_ID_SET_PATH = cls.CONTENT_PATH / TESTS_DIR / "id_set_mp_v2.json"
cls.XPANSE_ID_SET_PATH = cls.CONTENT_PATH / TESTS_DIR / "id_set_xpanse.json"
cls.LANDING_PAGE_SECTIONS_PATH = (
cls.CONTENT_PATH / TESTS_DIR / "Marketplace" / "landingPage_sections.json"
)
cls.NATIVE_IMAGE_PATH = cls.CONTENT_PATH / TESTS_DIR / NATIVE_IMAGE_FILE_NAME
cls.COMMON_SERVER_PYTHON_PATH = (
cls.CONTENT_PATH / "Packs" / "Base" / "Scripts" / "CommonServerPython"
)
cls.DEMISTO_MOCK_PATH = cls.CONTENT_PATH / TESTS_DIR / "demistomock"
cls.API_MODULES_SCRIPTS_DIR = (
cls.CONTENT_PATH / "Packs" / "ApiModules" / "Scripts"
)

cls.PYTHONPATH = [
path.absolute()
for path in [
cls.CONTENT_PATH,
cls.COMMON_SERVER_PYTHON_PATH,
cls.DEMISTO_MOCK_PATH,
Path(__file__).parent.parent / "lint" / "resources" / "pylint_plugins",
]
]

if cls.API_MODULES_SCRIPTS_DIR.exists():
cls.PYTHONPATH.extend(
path.absolute() for path in cls.API_MODULES_SCRIPTS_DIR.iterdir()
)
else:
logger.debug(
"Could not add API modules to 'PYTHONPATH' as the base directory does not exist."
)

PYTHONPATH_STR = ":".join(str(path) for path in PYTHONPATH)
cls.PYTHONPATH_STR = ":".join(str(path) for path in cls.PYTHONPATH)
10 changes: 5 additions & 5 deletions demisto_sdk/commands/common/errors.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
FileType,
MarketplaceVersions,
)
from demisto_sdk.commands.common.content_constant_paths import CONF_PATH
from demisto_sdk.commands.common.content_constant_paths import ContentPaths
from demisto_sdk.commands.common.tools import is_external_repository

FOUND_FILES_AND_ERRORS: list = []
Expand Down Expand Up @@ -2512,8 +2512,8 @@ def integration_not_registered(
file_path, missing_test_playbook_configurations, no_tests_key
):
return (
f"The following integration is not registered in {CONF_PATH} file.\n"
f"Please add:\n{missing_test_playbook_configurations}\nto {CONF_PATH} "
f"The following integration is not registered in {ContentPaths.CONF_PATH} file.\n"
f"Please add:\n{missing_test_playbook_configurations}\nto {ContentPaths.CONF_PATH} "
f"path under 'tests' key.\n"
f"If you don't want to add a test playbook for this integration, please add: \n{no_tests_key}to the "
f"file {file_path} or run 'demisto-sdk format -i {file_path}'"
Expand All @@ -2539,10 +2539,10 @@ def test_playbook_not_configured(
missing_integration_configurations,
):
return (
f"The TestPlaybook {content_item_id} is not registered in {CONF_PATH} file.\n "
f"The TestPlaybook {content_item_id} is not registered in {ContentPaths.CONF_PATH} file.\n "
f"Please add\n{missing_test_playbook_configurations}\n "
f"or if this test playbook is for an integration\n{missing_integration_configurations}\n "
f"to {CONF_PATH} path under 'tests' key."
f"to {ContentPaths.CONF_PATH} path under 'tests' key."
)

@staticmethod
Expand Down
12 changes: 8 additions & 4 deletions demisto_sdk/commands/common/hook_validations/conf_json.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
from pathlib import Path

from demisto_sdk.commands.common.constants import API_MODULES_PACK, FileType
from demisto_sdk.commands.common.content_constant_paths import CONF_PATH
from demisto_sdk.commands.common.content_constant_paths import ContentPaths
from demisto_sdk.commands.common.errors import Errors
from demisto_sdk.commands.common.handlers import DEFAULT_JSON_HANDLER as json
from demisto_sdk.commands.common.hook_validations.base_validator import (
Expand Down Expand Up @@ -37,7 +37,7 @@ def __init__(
self.conf_data = self.load_conf_file()

def load_conf_file(self):
with open(CONF_PATH) as data_file:
with open(ContentPaths.CONF_PATH) as data_file:
return json.load(data_file)

def is_valid_conf_json(self):
Expand Down Expand Up @@ -69,7 +69,9 @@ def is_valid_description_in_conf_dict(self, checked_dict):
error_message, error_code = Errors.description_missing_from_conf_json(
problematic_instances
)
if self.handle_error(error_message, error_code, file_path=CONF_PATH):
if self.handle_error(
error_message, error_code, file_path=ContentPaths.CONF_PATH
):
self._is_valid = False

return self._is_valid
Expand All @@ -91,7 +93,9 @@ def is_test_in_conf_json(self, file_id):
return True

error_message, error_code = Errors.test_not_in_conf_json(file_id)
if self.handle_error(error_message, error_code, file_path=CONF_PATH):
if self.handle_error(
error_message, error_code, file_path=ContentPaths.CONF_PATH
):
return False
return True

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
FileType,
)
from demisto_sdk.commands.common.content import Content
from demisto_sdk.commands.common.content_constant_paths import CONF_PATH, CONTENT_PATH
from demisto_sdk.commands.common.content_constant_paths import ContentPaths
from demisto_sdk.commands.common.errors import Errors
from demisto_sdk.commands.common.handlers import DEFAULT_JSON_HANDLER as json
from demisto_sdk.commands.common.handlers import DEFAULT_YAML_HANDLER as yaml
Expand Down Expand Up @@ -224,7 +224,7 @@ def is_valid_marketplaces_on_modified(self) -> bool:
try:
old_pack_marketplaces = set(
get_remote_file(
f"{CONTENT_PATH}/Packs/{pack_name}/pack_metadata.json",
f"{ContentPaths.CONTENT_PATH}/Packs/{pack_name}/pack_metadata.json",
tag=self.prev_ver,
).get(MARKETPLACE_KEY_PACK_METADATA, ())
)
Expand Down Expand Up @@ -368,7 +368,7 @@ def _is_id_equals_name(self, file_type):
return True

def _load_conf_file(self):
with open(CONF_PATH) as data_file:
with open(ContentPaths.CONF_PATH) as data_file:
return json.load(data_file)

@error_codes("CJ104,CJ102")
Expand Down
10 changes: 6 additions & 4 deletions demisto_sdk/commands/common/hook_validations/readme.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
RELATIVE_HREF_URL_REGEX,
RELATIVE_MARKDOWN_URL_REGEX,
)
from demisto_sdk.commands.common.content_constant_paths import CONTENT_PATH
from demisto_sdk.commands.common.content_constant_paths import ContentPaths
from demisto_sdk.commands.common.docker_helper import init_global_docker_client
from demisto_sdk.commands.common.errors import (
FOUND_FILES_AND_ERRORS,
Expand Down Expand Up @@ -173,7 +173,7 @@ def __init__(
json_file_path=json_file_path,
specific_validations=specific_validations,
)
self.content_path = CONTENT_PATH
self.content_path = ContentPaths.CONTENT_PATH
self.file_path_str = file_path
self.file_path = Path(file_path)
self.pack_path = self.file_path.parent
Expand Down Expand Up @@ -757,7 +757,9 @@ def empty_context_mgr(bool):
if mdx_server_is_up(): # this allows for this context to be reentrant
logger.debug("server is already up. Not restarting")
return empty_context_mgr(True)
if ReadMeValidator.are_modules_installed_for_verify(CONTENT_PATH): # type: ignore
if ReadMeValidator.are_modules_installed_for_verify(
ContentPaths.CONTENT_PATH
): # type: ignore
ReadMeValidator.add_node_env_vars()
return start_local_MDX_server(handle_error, file_path)
elif ReadMeValidator.is_docker_available():
Expand All @@ -766,7 +768,7 @@ def empty_context_mgr(bool):

@staticmethod
def add_node_env_vars():
content_path = CONTENT_PATH
content_path = ContentPaths.CONTENT_PATH
node_modules_path = content_path / Path("node_modules") # type: ignore
os.environ["NODE_PATH"] = (
str(node_modules_path) + os.pathsep + os.getenv("NODE_PATH", "")
Expand Down
Loading
Loading