diff --git a/coretex/cli/commands/task.py b/coretex/cli/commands/task.py
index 18a1e17e..93d1f2cc 100644
--- a/coretex/cli/commands/task.py
+++ b/coretex/cli/commands/task.py
@@ -16,6 +16,7 @@
# along with this program. If not, see .
from typing import Optional
+from pathlib import Path
import click
import webbrowser
@@ -24,13 +25,13 @@
from ..modules.project_utils import getProject
from ..modules.user import initializeUserSession
from ..modules.utils import onBeforeCommandExecute
-from ..modules.project_utils import getProject
+from ...networking import NetworkRequestError
from ..._folder_manager import folder_manager
from ..._task import TaskRunWorker, executeRunLocally, readTaskConfig, runLogger
from ...configuration import UserConfiguration
-from ...entities import TaskRun, TaskRunStatus
+from ...entities import Task, TaskRun, TaskRunStatus
+from ...entities.repository import checkIfCoretexRepoExists
from ...resources import PYTHON_ENTRY_POINT_PATH
-from ..._task import TaskRunWorker, executeRunLocally, readTaskConfig, runLogger
class RunException(Exception):
@@ -112,6 +113,45 @@ def run(path: str, name: Optional[str], description: Optional[str], snapshot: bo
folder_manager.clearTempFiles()
+@click.command()
+@click.argument("id", type = int, default = None, required = False)
+def pull(id: Optional[int]) -> None:
+ if id is None and not checkIfCoretexRepoExists():
+ id = ui.clickPrompt(f"There is no existing Task repository. Please specify id of Task you want to pull:", type = int)
+
+ if id is not None:
+ try:
+ task = Task.fetchById(id)
+ except NetworkRequestError as ex:
+ ui.errorEcho(f"Failed to fetch Task id {id}. Reason: {ex.response}")
+ return
+
+ if not task.coretexMetadataPath.exists():
+ task.pull()
+ task.createMetadata()
+ task.fillMetadata()
+ else:
+ differences = task.getDiff()
+ if len(differences) == 0:
+ ui.stdEcho("Your repository is already updated.")
+ return
+
+ ui.stdEcho("There are conflicts between your and remote repository.")
+ for diff in differences:
+ ui.stdEcho(f"File: {diff['path']} differs")
+ ui.stdEcho(f"\tLocal checksum: {diff['local_checksum']}")
+ ui.stdEcho(f"\tRemote checksum: {diff['remote_checksum']}")
+
+ if not ui.clickPrompt("Do you want to pull the changes and update your local repository? (Y/n):", type = bool, default = True):
+ ui.stdEcho("No changes were made to your local repository.")
+ return
+
+ task.pull()
+ task.createMetadata()
+ task.fillMetadata()
+ ui.stdEcho("Repository updated successfully.")
+
+
@click.group()
@onBeforeCommandExecute(initializeUserSession)
def task() -> None:
@@ -119,3 +159,4 @@ def task() -> None:
task.add_command(run, "run")
+task.add_command(pull, "pull")
\ No newline at end of file
diff --git a/coretex/entities/project/task.py b/coretex/entities/project/task.py
index 29e567a5..5b2fb181 100644
--- a/coretex/entities/project/task.py
+++ b/coretex/entities/project/task.py
@@ -20,10 +20,11 @@
from .base import BaseObject
from ..utils import isEntityNameValid
+from ..repository import CoretexRepository, EntityCoretexRepositoryType
from ...codable import KeyDescriptor
-class Task(BaseObject):
+class Task(BaseObject, CoretexRepository):
"""
Represents the task entity from Coretex.ai\n
@@ -33,6 +34,18 @@ class Task(BaseObject):
isDefault: bool
taskId: int
+ @property
+ def entityCoretexRepositoryType(self) -> EntityCoretexRepositoryType:
+ return EntityCoretexRepositoryType.task
+
+ @property
+ def paramKey(self) -> str:
+ return "sub_project_id"
+
+ @property
+ def endpoint(self) -> str:
+ return "workspace"
+
@classmethod
def _keyDescriptors(cls) -> Dict[str, KeyDescriptor]:
descriptors = super()._keyDescriptors()
diff --git a/coretex/entities/repository/__init__.py b/coretex/entities/repository/__init__.py
new file mode 100644
index 00000000..bea739e1
--- /dev/null
+++ b/coretex/entities/repository/__init__.py
@@ -0,0 +1 @@
+from .coretex_repository import CoretexRepository, EntityCoretexRepositoryType, checkIfCoretexRepoExists
diff --git a/coretex/entities/repository/coretex_repository.py b/coretex/entities/repository/coretex_repository.py
new file mode 100644
index 00000000..b382afad
--- /dev/null
+++ b/coretex/entities/repository/coretex_repository.py
@@ -0,0 +1,170 @@
+# Copyright (C) 2023 Coretex LLC
+
+# This file is part of Coretex.ai
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see .
+
+from typing import List, Dict, Any
+from enum import IntEnum
+from abc import abstractmethod, ABC
+from pathlib import Path
+from zipfile import ZipFile
+
+import os
+import json
+import logging
+
+from ...codable import Codable
+from ...utils import generateSha256Checksum
+from ...networking import networkManager, NetworkRequestError
+
+
+def checkIfCoretexRepoExists() -> bool:
+ print(Path.cwd().joinpath(".coretex"))
+ return Path.cwd().joinpath(".coretex").exists()
+
+
+class EntityCoretexRepositoryType(IntEnum):
+
+ task = 1
+
+
+class CoretexRepository(ABC, Codable):
+
+ id: int
+ projectId: int
+
+ @property
+ def __initialMetadataPath(self) -> Path:
+ return Path(f"{self.id}/.metadata.json")
+
+ @property
+ def coretexMetadataPath(self) -> Path:
+ return Path(f"{self.id}/.coretex.json")
+
+ @property
+ @abstractmethod
+ def entityCoretexRepositoryType(self) -> EntityCoretexRepositoryType:
+ pass
+
+ @property
+ @abstractmethod
+ def paramKey(self) -> str:
+ pass
+
+ @property
+ @abstractmethod
+ def endpoint(self) -> str:
+ pass
+
+ def pull(self) -> bool:
+ params = {
+ self.paramKey: self.id
+ }
+
+ zipFilePath = f"{self.id}.zip"
+ response = networkManager.download(f"{self.endpoint}/download", zipFilePath, params)
+
+ if response.hasFailed():
+ logging.getLogger("coretexpylib").error(f">> [Coretex] {self.entityCoretexRepositoryType.name.capitalize()} download has failed")
+ return False
+
+ with ZipFile(zipFilePath) as zipFile:
+ zipFile.extractall(str(self.id))
+
+ os.unlink(zipFilePath)
+
+ return not response.hasFailed()
+
+ def getRemoteMetadata(self) -> List:
+ # getRemoteMetadata downloads only .metadata file, this was needed so we can
+ # synchronize changes if multiple people work on the same Entity at the same time
+
+ params = {
+ self.paramKey: self.id
+ }
+
+ response = networkManager.get(f"{self.endpoint}/metadata", params)
+ if response.hasFailed():
+ raise NetworkRequestError(response, "Failed to fetch task metadata.")
+
+ return response.getJson(list, force = True)
+
+ def createMetadata(self) -> None:
+ # createMetadata() function will store metadata of files that backend returns
+ # currently all files on backend (that weren't uploaded after checksum calculation change
+ # for files that is implemented recently on backend) return null/None for their checksum
+ # if backend returns None for checksum of some file we need to calculate initial checksum of
+ # the file so we can track changes
+
+ with self.__initialMetadataPath.open("r") as initialMetadataFile:
+ initialMetadata = json.load(initialMetadataFile)
+
+ # if backend returns null for checksum of file, generate checksum
+ for file in initialMetadata:
+ if file["checksum"] is None:
+ filePath = self.__initialMetadataPath.parent.joinpath(file["path"])
+ if filePath.exists():
+ file["checksum"] = generateSha256Checksum(filePath)
+
+ newMetadata = {
+ "checksums": initialMetadata
+ }
+
+ with self.coretexMetadataPath.open("w") as coretexMetadataFile:
+ json.dump(newMetadata, coretexMetadataFile, indent = 4)
+
+ def fillMetadata(self) -> None:
+ # fillMetadata() function will update initial metadata returned from backend
+ # (file paths and their checksums) with other important Entity info (e.g. name, id, description...)
+
+ localMetadata: Dict[str, Any] = {}
+ metadata = self.encode()
+
+ with self.coretexMetadataPath.open("r") as coretexMetadataFile:
+ localMetadata = json.load(coretexMetadataFile)
+
+ localMetadata.update(metadata)
+
+ with self.coretexMetadataPath.open("w") as coretexMetadataFile:
+ json.dump(localMetadata, coretexMetadataFile, indent = 4)
+
+ self.__initialMetadataPath.unlink()
+
+ def getDiff(self) -> List[Dict[str, Any]]:
+ # getDiff() checks initial checksums of files stored in .coretex file
+ # and compares them with their current checksums, based on that we know what files have changes
+
+ with self.coretexMetadataPath.open("r") as localMetadataFile:
+ localMetadata = json.load(localMetadataFile)
+
+ localChecksums = {file["path"]: file["checksum"] for file in localMetadata["checksums"]}
+
+ differences = []
+
+ remoteMetadata = self.getRemoteMetadata()
+ for remoteFile in remoteMetadata:
+ remotePath = remoteFile["path"]
+ remoteChecksum = remoteFile["checksum"]
+
+ localChecksum = localChecksums.get(remotePath)
+
+ if localChecksum != remoteChecksum:
+ differences.append({
+ "path": remotePath,
+ "local_checksum": localChecksum,
+ "remote_checksum": remoteChecksum
+ })
+
+ return differences
diff --git a/coretex/entities/task_run/task_run.py b/coretex/entities/task_run/task_run.py
index f2263cb2..3889f957 100644
--- a/coretex/entities/task_run/task_run.py
+++ b/coretex/entities/task_run/task_run.py
@@ -17,7 +17,7 @@
from typing import Optional, Any, List, Dict, Union, Tuple, TypeVar, Generic, Type
from typing_extensions import Self, override
-from zipfile import ZipFile, ZIP_DEFLATED
+from zipfile import ZipFile
from pathlib import Path
import os
diff --git a/coretex/networking/network_response.py b/coretex/networking/network_response.py
index d187c1f2..6cc3c205 100644
--- a/coretex/networking/network_response.py
+++ b/coretex/networking/network_response.py
@@ -90,7 +90,7 @@ def isHead(self) -> bool:
return self._raw.request.method == RequestType.head.value
- def getJson(self, type_: Type[JsonType]) -> JsonType:
+ def getJson(self, type_: Type[JsonType], force: bool = False) -> JsonType:
"""
Converts HTTP response body to json
@@ -109,7 +109,7 @@ def getJson(self, type_: Type[JsonType]) -> JsonType:
TypeError -> If it was not possible to convert body to type of passed "type_" parameter
"""
- if not "application/json" in self.headers.get("Content-Type", ""):
+ if not force and not "application/json" in self.headers.get("Content-Type", ""):
raise ValueError(f">> [Coretex] Trying to convert request response to json but response \"Content-Type\" was \"{self.headers.get('Content-Type')}\"")
value = self._raw.json()
diff --git a/coretex/utils/__init__.py b/coretex/utils/__init__.py
index 80dd2753..b2788707 100644
--- a/coretex/utils/__init__.py
+++ b/coretex/utils/__init__.py
@@ -22,5 +22,5 @@
from .image import resizeWithPadding, cropToWidth
from .process import logProcessOutput, command, CommandException
from .logs import createFileHandler
-from .misc import isCliRuntime
+from .misc import isCliRuntime, generateSha256Checksum
from .error_handling import Throws
diff --git a/coretex/utils/misc.py b/coretex/utils/misc.py
index 9df00d44..f6b57aa8 100644
--- a/coretex/utils/misc.py
+++ b/coretex/utils/misc.py
@@ -15,8 +15,11 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see .
+from pathlib import Path
+
import os
import sys
+import hashlib
def isCliRuntime() -> bool:
@@ -25,3 +28,14 @@ def isCliRuntime() -> bool:
executablePath.endswith("/bin/coretex") and
os.access(executablePath, os.X_OK)
)
+
+
+def generateSha256Checksum(path: Path) -> str:
+ sha256 = hashlib.sha256()
+ chunkSize = 64 * 1024 # 65536 bytes
+
+ with path.open("rb") as file:
+ while chunk := file.read(chunkSize):
+ sha256.update(chunk)
+
+ return sha256.hexdigest()