Source Compatibility Suite: Run Cadence tests

compat/main.py

@@ -17,6 +17,7 @@
 from dacite import from_dict
 
 SUITE_PATH = Path("suite").resolve()
+CLI_PATH = SUITE_PATH / "flow-cli" / "cmd" / "flow"
 
 @contextmanager
 def cwd(path):
@@ -27,39 +28,78 @@ def cwd(path):
     finally:
         os.chdir(oldpwd)
 
+def cadence_replacement(cadence_version: Optional[str]) -> str:
+    if cadence_version:
+        return f'github.com/onflow/cadence@{cadence_version}'
+    # default: point to local cadence repo
+    return shlex.quote(Path.cwd().parent.absolute().resolve().as_posix())
+
+def flowgo_replacement(flowgo_version: Optional[str]) -> str:
+    if flowgo_version:
+        return f'github.com/onflow/flow-go@{flowgo_version}'
+    # default: use the newest version of flow-go available
+    return 'github.com/onflow/flow-go@latest'
+
+def build_cli(
+    cadence_version: Optional[str],
+    flowgo_version: Optional[str]
+):
+    logger.info("Building CLI ...")
+
+    with cwd(SUITE_PATH):
+        working_dir = SUITE_PATH / "flow-cli"
+        Git.clone("https://github.com/onflow/flow-cli.git", "master", working_dir)
+        with cwd(working_dir):
+            Go.replace_dependencies(
+                cadence_version=cadence_version,
+                flowgo_version=flowgo_version
+            )
+            logger.info("Compiling CLI binary")
+            subprocess.run(
+                ["make", "binary"],
+                check=True
+            )
+
+    logger.info("Built CLI")
+
 @dataclass
 class GoTest:
     path: str
     command: str
 
-    def run(self, working_dir: Path, prepare: bool, cadence_version: Optional[str], flowgo_version: Optional[str]) -> bool:
-        if cadence_version:
-            cadence_replacement = f'github.com/onflow/cadence@{cadence_version}'
-        else:
-            # default: point to local cadence repo
-            cadence_replacement = shlex.quote(Path.cwd().parent.absolute().resolve().as_posix())
-
-        if flowgo_version:
-            flowgo_replacement = f'github.com/onflow/flow-go@{flowgo_version}'
-        else:
-            # default: use the newest version of flow-go available
-            flowgo_replacement = 'github.com/onflow/flow-go@latest'
+    def run(
+        self,
+        working_dir: Path,
+        prepare: bool,
+        cadence_version: Optional[str],
+        flowgo_version: Optional[str]
+    ) -> bool:
 
         with cwd(working_dir / self.path):
             if prepare:
-                logger.info("Editing dependencies")
-                subprocess.run([
-                    "go", "get", flowgo_replacement,
-                ])
-                subprocess.run([
-                    "go", "mod", "edit", "-replace", f'github.com/onflow/cadence={cadence_replacement}',
-                ])
-                logger.info("Downloading dependencies")
-                subprocess.run([
-                    "go", "get", "-t", ".",
-                ])
-
-            result = subprocess.run(shlex.split(self.command))
+                Go.replace_dependencies(cadence_version, flowgo_version)
+
+            result = subprocess.run(self.command, shell=True)
+            return result.returncode == 0
+
+@dataclass
+class CadenceTest:
+    path: str
+    command: str
+
+    def run(
+        self,
+        working_dir: Path,
+        prepare: bool,
+        cadence_version: Optional[str],
+        flowgo_version: Optional[str]
+    ) -> bool:
+
+        env = os.environ.copy()
+        env["PATH"] = f"{shlex.quote(str(CLI_PATH))}:{env['PATH']}"
+
+        with cwd(working_dir / self.path):
+            result = subprocess.run(self.command, shell=True, env=env)
             return result.returncode == 0
 
 def load_index(path: Path) -> List[str]:
@@ -73,6 +113,7 @@ class Description:
     url: str
     branch: str
     go_tests: List[GoTest] = field(default_factory=list)
+    cadence_tests: List[CadenceTest] = field(default_factory=list)
 
     @staticmethod
     def load(name: str) -> Description:
@@ -85,69 +126,105 @@ def load(name: str) -> Description:
     def from_dict(cls, data: Dict):
         return from_dict(data_class=cls, data=data)
 
-    def _clone(self, working_dir: Path):
-        if working_dir.exists():
-            for root, dirs, files in os.walk(working_dir):
-                for dir in dirs:
-                    os.chmod(os.path.join(root, dir), stat.S_IRUSR | stat.S_IWUSR)
-                for file in files:
-                    os.chmod(os.path.join(root, file), stat.S_IRUSR | stat.S_IWUSR)
-            shutil.rmtree(working_dir)
-
-        logger.info(f"Cloning {self.url} ({self.branch})")
-
-        Git.clone(self.url, self.branch, working_dir)
-
     def run(
         self,
         name: str,
         prepare: bool,
         go_test: bool,
+        cadence_test: bool,
         cadence_version: Optional[str],
         flowgo_version: Optional[str],
     ) -> (bool):
 
+        logger.info(f"Running tests for {name} ...")
+
         working_dir = SUITE_PATH / name
 
         if prepare:
-            self._clone(working_dir)
+            Git.clone(self.url, self.branch, working_dir)
 
         go_tests_succeeded = True
         if go_test:
             for test in self.go_tests:
-                if not test.run(working_dir, prepare=prepare, cadence_version=cadence_version, flowgo_version=flowgo_version):
+                if not test.run(
+                    working_dir,
+                    prepare=prepare,
+                    cadence_version=cadence_version,
+                    flowgo_version=flowgo_version
+                ):
                     go_tests_succeeded = False
 
-        succeeded = go_tests_succeeded
+        cadence_tests_succeeded = True
+        if cadence_test:
+            for test in self.cadence_tests:
+                if not test.run(
+                    working_dir,
+                    prepare=prepare,
+                    cadence_version=cadence_version,
+                    flowgo_version=flowgo_version
+                ):
+                    cadence_tests_succeeded = False
+
+        return go_tests_succeeded and cadence_tests_succeeded
 
-        return succeeded
+class Go:
+
+    @staticmethod
+    def mod_replace(original: str, replacement: str):
+        subprocess.run(
+            ["go", "mod", "edit", "-replace", f'{original}={replacement}'],
+            check=True
+        )
+
+    @staticmethod
+    def mod_tidy():
+        subprocess.run(
+            ["go", "mod", "tidy"],
+            check=True
+        )
+
+    @staticmethod
+    def replace_dependencies(
+        cadence_version: Optional[str],
+        flowgo_version: Optional[str]
+    ):
+        logger.info("Editing dependencies")
+        Go.mod_replace("github.com/onflow/cadence", cadence_replacement(cadence_version))
+        Go.mod_replace("github.com/onflow/flow-go", flowgo_replacement(flowgo_version))
+        Go.mod_tidy()
 
 class Git:
 
     @staticmethod
     def clone(url: str, branch: str, working_dir: Path):
-        subprocess.run([
-            "git", "clone", "--depth", "1", "--branch",
-            branch, url, working_dir
-        ])
+        if working_dir.exists():
+            Git._clean(working_dir)
+
+        logger.info(f"Cloning {url} ({branch})")
+
+        subprocess.run(
+            ["git", "clone", "--depth", "1", "--branch", branch, url, working_dir],
+            check=True
+        )
+
+    @staticmethod
+    def _clean(working_dir: Path):
+        for root, dirs, files in os.walk(working_dir):
+            for dir in dirs:
+                os.chmod(os.path.join(root, dir), stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
+            for file in files:
+                os.chmod(os.path.join(root, file), stat.S_IRUSR | stat.S_IWUSR)
+        shutil.rmtree(working_dir)
 
     @staticmethod
     def get_head_ref() -> str:
         completed_process = subprocess.run(
             ["git", "rev-parse", "--abbrev-ref", "HEAD"],
             capture_output=True,
+            check=True
         )
-        if completed_process.returncode != 0:
-            raise Exception('failed to get current Git ref')
         return completed_process.stdout.decode("utf-8").strip()
 
-    @staticmethod
-    def checkout(ref: str):
-        completed_process = subprocess.run(["git", "checkout", ref])
-        if completed_process.returncode != 0:
-            raise Exception(f'failed to checkout ref {ref}')
-
-
 @click.command()
 @click.option(
     "--rerun",
@@ -161,6 +238,12 @@ def checkout(ref: str):
     default=True,
     help="Run the suite Go tests"
 )
+@click.option(
+    "--cadence-test/--no-cadence-test",
+    is_flag=True,
+    default=True,
+    help="Run the suite Cadence tests"
+)
 @click.option(
     "--cadence-version",
     default=None,
@@ -176,20 +259,34 @@ def checkout(ref: str):
     nargs=-1,
 )
 def main(
-        rerun: bool,
-        go_test: bool,
-        cadence_version: str,
-        flowgo_version: str,
-        names: Collection[str]
+    rerun: bool,
+    go_test: bool,
+    cadence_test: bool,
+    cadence_version: str,
+    flowgo_version: str,
+    names: Collection[str]
 ):
 
+    logger.info(
+        f'Chosen versions: '
+        + f'cadence@{ cadence_version if cadence_version else "local version" }, '
+        + f'flow-go@{flowgo_version if flowgo_version else "latest"}'
+    )
+
     prepare = not rerun
 
+    if cadence_test and prepare:
+        build_cli(
+            cadence_version=cadence_version,
+            flowgo_version=flowgo_version,
+        )
+
     # Run for the current checkout
 
     current_success = run(
         prepare=prepare,
         go_test=go_test,
+        cadence_test=cadence_test,
         cadence_version=cadence_version,
         flowgo_version=flowgo_version,
         names=names
@@ -200,17 +297,16 @@ def main(
 
 
 def run(
-        prepare: bool,
-        go_test: bool,
-        cadence_version: str,
-        flowgo_version: str,
-        names: Collection[str]
+    prepare: bool,
+    go_test: bool,
+    cadence_test: bool,
+    cadence_version: str,
+    flowgo_version: str,
+    names: Collection[str]
 ) -> (bool):
 
     all_succeeded = True
 
-    logger.info(f'Chosen versions: cadence@{ cadence_version if cadence_version else "local version" }, flow-go@{flowgo_version if flowgo_version else "latest"}')
-
     if not names:
         names = load_index(SUITE_PATH / "index.yaml")
 
@@ -222,6 +318,7 @@ def run(
             name,
             prepare=prepare,
             go_test=go_test,
+            cadence_test=cadence_test,
             cadence_version=cadence_version,
             flowgo_version=flowgo_version,
         )

compat/suite/flow-core-contracts.yaml

@@ -7,3 +7,6 @@ branch: master
 go_tests:
 - path: lib/go/test
   command: make test
+cadence_tests:
+- path: .
+  command: flow test tests/*.cdc

compat/suite/green-goo-dao-flow-utils.yaml

@@ -0,0 +1,8 @@
+description: Green Goo Dao flow-utils
+maintainers:
+  - [email protected]
+url: https://github.com/turbolent/flow-utils.git
+branch: improvements
+cadence_tests:
+- path: .
+  command: npm i && ./run-tests.sh