Skip to content

Commit

Permalink
feat: Rework artifact format, lots of renaming
Browse files Browse the repository at this point in the history
BREAKING CHANGE: Lots of format changes during the ci inter-communication
  • Loading branch information
FHeilmann committed Dec 18, 2023
1 parent cfba786 commit 488a0cf
Show file tree
Hide file tree
Showing 11 changed files with 408 additions and 307 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/post_process_pr.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ jobs:
steps:
- name: Create PR comment 💬
id: create_pr_comment
uses: docker://ghcr.io/fheilmann/voron_toolkit_docker:latest
uses: docker://ghcr.io/vorondesign/voron_toolkit_docker:latest
env:
PR_HELPER_WORKFLOW_RUN_ID: ${{ github.event.workflow_run.id }}
PR_HELPER_ARTIFACT_NAME: ci_output
Expand All @@ -30,7 +30,7 @@ jobs:
steps:
- name: Upload images to imagekit 📸
id: imagekit-upload
uses: docker://ghcr.io/fheilmann/voron_toolkit_docker:latest
uses: docker://ghcr.io/vorondesign/voron_toolkit_docker:latest
env:
IMAGEKIT_UPLOADER_WORKFLOW_RUN_ID: ${{ github.event.workflow_run.id }}
IMAGEKIT_UPLOADER_ARTIFACT_NAME: ci_output
Expand Down
130 changes: 107 additions & 23 deletions voron_toolkit/constants.py
Original file line number Diff line number Diff line change
@@ -1,40 +1,124 @@
import json
from collections import defaultdict
from dataclasses import dataclass
from enum import Enum
from typing import NamedTuple
from typing import Any, NamedTuple, Self

CI_PASSED_LABEL: str = "CI: Passed"
CI_FAILURE_LABEL: str = "CI: Issues identified"
CI_ERROR_LABEL: str = "Warning: CI Error"
PR_COMMENT_TAG: str = "<!-- voron_docker_toolkit -->"


class StepResultCodeStr(NamedTuple):
result_code: int
result_icon: str
class ExtendedResult(NamedTuple):
code: int
icon: str


class StepIdName(NamedTuple):
step_id: str
step_name: str
class ExtendedResultEnum(ExtendedResult, Enum):
SUCCESS = ExtendedResult(code=0, icon="✅")
WARNING = ExtendedResult(code=1, icon="⚠️")
FAILURE = ExtendedResult(code=2, icon="❌")
EXCEPTION = ExtendedResult(code=3, icon="💀")


class StepResult(StepResultCodeStr, Enum):
SUCCESS = StepResultCodeStr(result_code=0, result_icon="✅")
WARNING = StepResultCodeStr(result_code=1, result_icon="⚠️")
FAILURE = StepResultCodeStr(result_code=2, result_icon="❌")
EXCEPTION = StepResultCodeStr(result_code=3, result_icon="💀")
class ItemResult(NamedTuple):
item: str
extra_info: list[str]


class StepIdentifier(StepIdName, Enum):
CORRUPTION_CHECK = StepIdName(step_id="corruption_check", step_name="STL corruption checker")
MOD_STRUCTURE_CHECK = StepIdName(step_id="mod_structure_check", step_name="Mod structure checker")
README_GENERATOR = StepIdName(step_id="readme_generator", step_name="Readme generator")
ROTATION_CHECK = StepIdName(step_id="rotation_check", step_name="STL rotation checker")
WHITESPACE_CHECK = StepIdName(step_id="whitespace_check", step_name="Whitespace checker")
@dataclass
class ToolSummaryTable:
extra_columns: list[str]
items: defaultdict[ExtendedResultEnum, list[ItemResult]]

def to_markdown(self: Self, filter_result: ExtendedResultEnum | None = None) -> str:
if filter_result:
rows = [[row.item, *row.extra_info] for row in self.items[filter_result]]
markdown: str = self.create_markdown_table(columns=["Item", *self.extra_columns], rows=rows) if rows else ""
else:
rows = [[row.item, f"{result.icon} {result.name}", *row.extra_info] for result in self.items for row in self.items[result]]
markdown = self.create_markdown_table(columns=["Item", "Result", *self.extra_columns], rows=rows) if rows else ""
return f"{markdown}"

VORONUSERS_PR_COMMENT_SECTIONS: list[StepIdentifier] = [
StepIdentifier.WHITESPACE_CHECK,
StepIdentifier.MOD_STRUCTURE_CHECK,
StepIdentifier.CORRUPTION_CHECK,
StepIdentifier.ROTATION_CHECK,
@classmethod
def _create_table_header(cls: type[Self], columns: list[str]) -> str:
column_names = "| " + " | ".join(columns) + " |"
dividers = "| " + " | ".join(["---"] * len(columns)) + " |"
return f"{column_names}\n{dividers}"

@classmethod
def _create_markdown_table_rows(cls: type[Self], rows: list[list[str]]) -> str:
return "\n".join(["| " + " | ".join(row) + " |" for row in rows])

@classmethod
def create_markdown_table(cls: type[Self], columns: list[str], rows: list[list[str]]) -> str:
return f"{cls._create_table_header(columns=columns)}\n{cls._create_markdown_table_rows(rows=rows)}\n"


@dataclass
class ToolResult:
tool_id: str
tool_name: str
extended_result: ExtendedResultEnum
tool_ignore_warnings: bool
tool_result_items: ToolSummaryTable

def to_json(self: Self) -> str:
dct: dict[str, Any] = {
"tool_id": self.tool_id,
"tool_name": self.tool_name,
"extended_result": self.extended_result.name,
"tool_ignore_warnings": self.tool_ignore_warnings,
"tool_result_items_extra_columns": self.tool_result_items.extra_columns,
}
dct["tool_result_items"] = {}
for extended_result in ExtendedResultEnum:
dct["tool_result_items"][extended_result.name] = [
{"item": itemresult.item, "extra_info": itemresult.extra_info} for itemresult in self.tool_result_items.items[extended_result]
]
return json.dumps(dct, indent=4)

@classmethod
def from_json(cls: type[Self], json_string: str) -> Self:
dct = json.loads(json_string)
return cls(
tool_id=dct.get("tool_id"),
tool_name=dct.get("tool_name"),
extended_result=ExtendedResultEnum[dct.get("extended_result")],
tool_ignore_warnings=dct.get("tool_ignore_warnings"),
tool_result_items=ToolSummaryTable(
extra_columns=dct.get("tool_result_items_extra_columns"),
items=defaultdict(
list,
{
extended_result: [
ItemResult(item=item_result["item"], extra_info=item_result["extra_info"])
for item_result in dct.get("tool_result_items").get(extended_result.name)
]
for extended_result in ExtendedResultEnum
},
),
),
)


class ToolIdentifier(NamedTuple):
tool_id: str
tool_name: str


class ToolIdentifierEnum(ToolIdentifier, Enum):
CORRUPTION_CHECK = ToolIdentifier(tool_id="corruption_check", tool_name="STL corruption checker")
MOD_STRUCTURE_CHECK = ToolIdentifier(tool_id="mod_structure_check", tool_name="Mod structure checker")
README_GENERATOR = ToolIdentifier(tool_id="readme_generator", tool_name="Readme generator")
ROTATION_CHECK = ToolIdentifier(tool_id="rotation_check", tool_name="STL rotation checker")
WHITESPACE_CHECK = ToolIdentifier(tool_id="whitespace_check", tool_name="Whitespace checker")


VORONUSERS_PR_COMMENT_SECTIONS: list[ToolIdentifierEnum] = [
ToolIdentifierEnum.WHITESPACE_CHECK,
ToolIdentifierEnum.MOD_STRUCTURE_CHECK,
ToolIdentifierEnum.CORRUPTION_CHECK,
ToolIdentifierEnum.ROTATION_CHECK,
]
124 changes: 60 additions & 64 deletions voron_toolkit/tools/mod_structure_checker.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import json
from collections import defaultdict
from enum import StrEnum
from importlib.resources import files
from pathlib import Path
Expand All @@ -10,19 +11,18 @@
from loguru import logger

from voron_toolkit import resources
from voron_toolkit.constants import StepIdentifier, StepResult
from voron_toolkit.utils.action_summary import ActionSummaryTable
from voron_toolkit.constants import ExtendedResultEnum, ItemResult, ToolIdentifierEnum, ToolResult, ToolSummaryTable
from voron_toolkit.utils.file_helper import FileHelper
from voron_toolkit.utils.github_action_helper import ActionResult, GithubActionHelper
from voron_toolkit.utils.github_action_helper import GithubActionHelper
from voron_toolkit.utils.logging import init_logging


class FileErrors(StrEnum):
file_from_metadata_missing = "The file '{}' is listed in the metadata.yml file but does not exist"
file_outside_mod_folder = "The file '{}' is located outside the expected folder structure of `printer_mods/user/mod`"
mod_has_no_cad_files = "The mod '{}' does not have any CAD files listed in the metadata.yml file"
mod_missing_metadata = "The mod '{}' does not have a metadata.yml file"
mod_has_invalid_metadata_file = "The metadata file of mod '{}' is invalid!"
file_from_metadata_missing = "The file is listed in the metadata.yml file but does not exist"
file_outside_mod_folder = "The file is located outside the expected folder structure of `printer_mods/user/mod`"
mod_has_no_cad_files = "The mod does not have any CAD files listed in the metadata.yml file"
mod_missing_metadata = "The mod does not have a metadata.yml file"
mod_has_invalid_metadata_file = "The metadata file of mod is invalid!"


IGNORE_FILES = ["README.md", "mods.json"]
Expand All @@ -33,67 +33,64 @@ class FileErrors(StrEnum):
class ModStructureChecker:
def __init__(self: Self, args: configargparse.Namespace) -> None:
self.input_dir: Path = Path(Path.cwd(), args.input_dir)
self.gh_helper: GithubActionHelper = GithubActionHelper(ignore_warnings=args.ignore_warnings)
self.return_status: StepResult = StepResult.SUCCESS
self.check_summary: list[list[str]] = []
self.gh_helper: GithubActionHelper = GithubActionHelper()
self.ignore_warnings = args.ignore_warnings
self.return_status: ExtendedResultEnum = ExtendedResultEnum.SUCCESS
self.result_items: defaultdict[ExtendedResultEnum, list[ItemResult]] = defaultdict(list)

init_logging(verbose=args.verbose)

def _check_mods(self: Self) -> None:
mod_folders = [folder for folder in self.input_dir.glob("*/*") if folder.is_dir()]
logger.info("Performing mod structure and metadata check")
result: StepResult = StepResult.SUCCESS
result: ExtendedResultEnum = ExtendedResultEnum.SUCCESS
schema = json.loads(files(resources).joinpath("voronusers_metadata_schema.json").read_text())
for mod_folder in mod_folders:
mod_folder_relative: str = mod_folder.relative_to(self.input_dir).as_posix()
if not Path(mod_folder, ".metadata.yml").exists():
logger.error("Mod '{}' is missing a metadata file!", mod_folder_relative)
self.check_summary.append(
[
mod_folder_relative,
f"{StepResult.FAILURE.result_icon} {StepResult.FAILURE.name}",
FileErrors.mod_missing_metadata.value.format(mod_folder_relative),
]
self.result_items[ExtendedResultEnum.FAILURE].append(
ItemResult(
item=mod_folder_relative,
extra_info=[FileErrors.mod_missing_metadata.value],
)
)
result = StepResult.FAILURE
result = ExtendedResultEnum.FAILURE
continue

try:
metadata: dict[str, Any] = yaml.safe_load(Path(mod_folder, ".metadata.yml").read_text())
jsonschema.validate(instance=metadata, schema=schema)
except (yaml.YAMLError, yaml.scanner.ScannerError) as e:
logger.error("YAML error in metadata file of mod '{}': {}", mod_folder, e)
self.check_summary.append(
[
Path(mod_folder, ".metadata.yml").relative_to(self.input_dir).as_posix(),
f"{StepResult.FAILURE.result_icon} {StepResult.FAILURE.name}",
FileErrors.mod_has_invalid_metadata_file.value.format(mod_folder),
]
self.result_items[ExtendedResultEnum.FAILURE].append(
ItemResult(
item=Path(mod_folder, ".metadata.yml").relative_to(self.input_dir).as_posix(),
extra_info=[FileErrors.mod_has_invalid_metadata_file.value],
)
)
result = StepResult.FAILURE
result = ExtendedResultEnum.FAILURE
continue
except jsonschema.ValidationError as e:
logger.error("Validation error in metadata file of mod '{}': {}", mod_folder, e.message)
self.check_summary.append(
[
Path(mod_folder, ".metadata.yml").relative_to(self.input_dir).as_posix(),
f"{StepResult.FAILURE.result_icon} {StepResult.FAILURE.name}",
FileErrors.mod_has_invalid_metadata_file.value.format(mod_folder_relative),
]
self.result_items[ExtendedResultEnum.FAILURE].append(
ItemResult(
item=Path(mod_folder, ".metadata.yml").relative_to(self.input_dir).as_posix(),
extra_info=[FileErrors.mod_has_invalid_metadata_file.value],
)
)
result = StepResult.FAILURE
result = ExtendedResultEnum.FAILURE
continue

if "cad" in metadata and not metadata["cad"]:
logger.warning("Mod '{}' has no CAD files!", mod_folder)
self.check_summary.append(
[
mod_folder_relative,
f"{StepResult.FAILURE.result_icon} {StepResult.FAILURE.name}",
FileErrors.mod_has_no_cad_files.value.format(mod_folder_relative),
]
self.result_items[ExtendedResultEnum.FAILURE].append(
ItemResult(
item=mod_folder_relative,
extra_info=[FileErrors.mod_has_no_cad_files.value],
)
)
result = StepResult.FAILURE
result = ExtendedResultEnum.FAILURE

for subelement in ["cad", "images"]:
metadata_files = metadata[subelement]
Expand All @@ -102,33 +99,31 @@ def _check_mods(self: Self) -> None:
for metadata_file in metadata_files:
if not Path(mod_folder, metadata_file).exists():
logger.error("File '{}' is missing in mod folder '{}'!", metadata_file, mod_folder_relative)
self.check_summary.append(
[
mod_folder_relative,
f"{StepResult.FAILURE.result_icon} {StepResult.FAILURE.name}",
FileErrors.file_from_metadata_missing.value.format(metadata_file),
]
self.result_items[ExtendedResultEnum.FAILURE].append(
ItemResult(
item=mod_folder_relative,
extra_info=[FileErrors.file_from_metadata_missing.value],
)
)
result = StepResult.FAILURE
self.check_summary.append([mod_folder_relative, f"{StepResult.SUCCESS.result_icon} {StepResult.SUCCESS.name}", ""])
result = ExtendedResultEnum.FAILURE
self.result_items[ExtendedResultEnum.SUCCESS].append(ItemResult(item=mod_folder_relative, extra_info=[""]))
logger.success("Folder '{}' OK!", mod_folder_relative)
self.return_status = result

def _check_shallow_files(self: Self) -> None:
logger.info("Performing shallow file check")
files_folders = FileHelper.get_shallow_folders(input_dir=self.input_dir, max_depth=MOD_DEPTH - 1, ignore=IGNORE_FILES)
result: StepResult = StepResult.SUCCESS
result: ExtendedResultEnum = ExtendedResultEnum.SUCCESS
for file_folder in files_folders:
logger.error("File '{}' outside mod folder structure!", file_folder)
self.check_summary.append(
[
file_folder.relative_to(self.input_dir).as_posix(),
f"{StepResult.FAILURE.result_icon} {StepResult.FAILURE.name}",
FileErrors.file_outside_mod_folder.value.format(file_folder),
]
self.result_items[ExtendedResultEnum.FAILURE].append(
ItemResult(
item=file_folder.relative_to(self.input_dir).as_posix(),
extra_info=[FileErrors.file_outside_mod_folder.value],
)
)
result = StepResult.FAILURE
if result == StepResult.SUCCESS:
result = ExtendedResultEnum.FAILURE
if result == ExtendedResultEnum.SUCCESS:
logger.success("Shallow file check OK!")
self.return_status = result

Expand All @@ -140,13 +135,14 @@ def run(self: Self) -> None:
self._check_mods()

self.gh_helper.finalize_action(
action_result=ActionResult(
action_id=StepIdentifier.MOD_STRUCTURE_CHECK.step_id,
action_name=StepIdentifier.MOD_STRUCTURE_CHECK.step_name,
outcome=self.return_status,
summary=ActionSummaryTable(
columns=["File/Folder", "Result", "Reason"],
rows=self.check_summary,
action_result=ToolResult(
tool_id=ToolIdentifierEnum.MOD_STRUCTURE_CHECK.tool_id,
tool_name=ToolIdentifierEnum.MOD_STRUCTURE_CHECK.tool_name,
extended_result=self.return_status,
tool_ignore_warnings=self.ignore_warnings,
tool_result_items=ToolSummaryTable(
extra_columns=["Reason"],
items=self.result_items,
),
)
)
Expand Down
Loading

0 comments on commit 488a0cf

Please sign in to comment.