Fixed project lookup of classes when resolving source only projects (#18

)

* Fixed project lookup of classes when resolving source only projects

* Docker images need to be created when the branch name is known

* libcrypto.so is not required

.github/workflows/docker-images.yml

@@ -13,8 +13,7 @@ on:
       - '.github/workflows/*.sh'
   # need images created at least once per branch, even if there are no docker changes
   # so that downstream projects can use the branch channel
-  pull_request:
-    types: [opened]
+  create:
   # Make new images for every published release
   release:
     types: [published]

.github/workflows/sanity-test.yml

@@ -107,6 +107,7 @@ jobs:
       - name: Sanity Test Run Compat (Unix)
         if:  matrix.host != 'windows-latest'
         run: |
+          ls -al
           zipinfo -1 builder.pyz
           python3 builder.pyz run test --project tests
 

builder/__main__.py

@@ -223,6 +223,8 @@ def parse_args():
         'spec': spec,
     })
 
+    Scripts.load()
+
     if env.spec.target == current_os() and env.spec.arch == current_arch():
         inspect_host(env)
     if args.command == 'inspect':

builder/imports/libcrypto.py

@@ -46,13 +46,14 @@ def install(self, env):
             required_files = [
                 ['include/openssl/crypto.h'],
                 ['lib/libcrypto.a', 'lib64/libcrypto.a'],
-                ['lib/libcrypto.so', 'lib64/libcrypto.so'],
+                #['lib/libcrypto.so', 'lib64/libcrypto.so'],
             ]
             found = 0
             for paths in required_files:
                 for path in paths:
                     full_path = os.path.join(self.prefix, path)
                     if os.path.isfile(full_path):
+                        print('Found {}'.format(full_path))
                         found += 1
                         break
 

builder/imports/s2n.py

@@ -14,7 +14,7 @@
 from project import Project, Import
 
 
-class S2N(Project):
+class S2N(Project, Import):
     def __init__(self, **kwargs):
         super().__init__(
             account='awslabs',
@@ -27,16 +27,3 @@ def __init__(self, **kwargs):
                 },
             },
             **kwargs)
-
-
-class S2N(Import):
-    def __init__(self, **kwargs):
-        super().__init__(
-            imports=['libcrypto'],
-            config={
-                'targets': ['linux'],
-                'cmake_args': {
-                    '-DS2N_NO_PQ_ASM=ON',
-                },
-            },
-            **kwargs)

builder/project.py

@@ -331,7 +331,8 @@ class Project(object):
 
     def __init__(self, **kwargs):
         self.account = kwargs.get('account', 'awslabs')
-        self.name = kwargs['name']
+        self.name = kwargs.get('name', self.__class__.__name__.lower())
+        assert self.name != 'project'
         self.url = kwargs.get('url', "https://github.com/{}/{}.git".format(
             self.account, self.name))
         self.path = kwargs.get('path', None)
@@ -596,10 +597,10 @@ def find_project(name, hints=[]):
                 # might be a project without a config
                 if looks_like_code(search_dir):
                     print(
-                        ('    Found source code that looks like a project at {}'.format(search_dir)))
-                    project = Project._cache_project(
-                        Project(name=name, path=search_dir))
-                    return project
+                        ('    Found source code only project at {}'.format(search_dir)))
+                    project = Project._create_project(
+                        name=name, path=search_dir)
+                    return Project._cache_project(project)
 
         if Project._find_project_class(name):
             return Project._cache_project(Project._create_project(name))

builder/scripts.py

@@ -12,6 +12,7 @@
 # permissions and limitations under the License.
 
 import importlib
+import importlib.util
 import glob
 import os
 import sys
@@ -52,50 +53,46 @@ class Scripts(object):
     @staticmethod
     def load(path='.'):
         """ Loads all scripts from ${path}/.builder/**/*.py to make their classes available """
-        import importlib.util
 
+        # Load any classes from path
         path = os.path.abspath(os.path.join(path, '.builder'))
-        if not os.path.isdir(path):
-            return
-
-        print('Loading scripts from {}'.format(path))
-        scripts = glob.glob(os.path.join(path, '*.py'))
-        scripts += glob.glob(os.path.join(path, '**', '*.py'))
-
-        # Update to get the latest action set right before we load
-        existing_classes = _get_all_dynamic_classes()
-        for script in scripts:
-            if not script.endswith('.py'):
-                continue
-
-            # Ensure that the import path includes the directory the script is in
-            # so that relative imports work
-            script_dir = os.path.dirname(script)
-            if script_dir not in sys.path:
-                sys.path.append(script_dir)
-            print("Importing {}".format(os.path.abspath(script)), flush=True)
-
-            name = os.path.split(script)[1].split('.')[0]
-            spec = importlib.util.spec_from_file_location(name, script)
-            module = importlib.util.module_from_spec(spec)
-            spec.loader.exec_module(module)
-            # Must invalidate caches or sometimes the loaded classes won't be found
-            # See: https://docs.python.org/3/library/importlib.html#importlib.invalidate_caches
-            importlib.invalidate_caches()
-
-            # Report newly loaded actions
-            classes = frozenset(_get_all_dynamic_classes())
-            new_classes = classes.difference(existing_classes)
-            if new_classes:
-                print("Imported {}".format(
-                    ', '.join([c.__name__ for c in new_classes])))
-                Scripts.all_classes.update(new_classes)
+        if os.path.isdir(path):
+            print('Loading scripts from {}'.format(path))
+            scripts = glob.glob(os.path.join(path, '*.py'))
+            scripts += glob.glob(os.path.join(path, '**', '*.py'))
+
+            for script in scripts:
+                if not script.endswith('.py'):
+                    continue
+
+                # Ensure that the import path includes the directory the script is in
+                # so that relative imports work
+                script_dir = os.path.dirname(script)
+                if script_dir not in sys.path:
+                    sys.path.append(script_dir)
+                print("Importing {}".format(os.path.abspath(script)), flush=True)
+
+                name = os.path.split(script)[1].split('.')[0]
+                spec = importlib.util.spec_from_file_location(name, script)
+                module = importlib.util.module_from_spec(spec)
+                spec.loader.exec_module(module)
+                # Must invalidate caches or sometimes the loaded classes won't be found
+                # See: https://docs.python.org/3/library/importlib.html#importlib.invalidate_caches
+                importlib.invalidate_caches()
+
+        # Report newly loaded classes
+        classes = frozenset(_get_all_dynamic_classes())
+        new_classes = classes.difference(Scripts.all_classes)
+        if new_classes:
+            print("Imported {}".format(
+                ', '.join([c.__name__ for c in new_classes])))
+            Scripts.all_classes.update(new_classes)
 
     @staticmethod
     def _find_actions():
         _import_dynamic_classes()
-        _all_actions = set(Action.__subclasses__())
-        return _all_actions
+        actions = set(Action.__subclasses__())
+        return actions
 
     @staticmethod
     def find_action(name):