feat: support for similar function

Signed-off-by: vsoch <[email protected]>

.github/CONTRIBUTING.md

@@ -4,10 +4,10 @@ This code is licensed under the MPL 2.0 [LICENSE](LICENSE).
 
 # Contributing
 
-When contributing to Singularity Registry Global Client, it is 
-important to properly communicate the gist of the contribution. 
-If it is a simple code or editorial fix, simply explaining this 
-within the GitHub Pull Request (PR) will suffice. But if this is a larger 
+When contributing to Singularity Registry Global Client, it is
+important to properly communicate the gist of the contribution.
+If it is a simple code or editorial fix, simply explaining this
+within the GitHub Pull Request (PR) will suffice. But if this is a larger
 fix or Enhancement, it should be first discussed with the project
 leader or developers.
 
@@ -29,8 +29,8 @@ all your interactions with the project members and users.
 4. The project's default copyright and header have been included in any new
    source files.
 5. All (major) changes to Singularity Registry must be documented in
-   [docs](docs). If your PR changes a core functionality, please 
-   include clear description of the changes in your PR so that the docs 
+   [docs](docs). If your PR changes a core functionality, please
+   include clear description of the changes in your PR so that the docs
    can be updated, or better, submit another PR to update the docs directly.
 6. If necessary, update the README.md.
 7. The pull request will be reviewed by others, and the final merge must be
@@ -102,7 +102,7 @@ an incident. Further details of specific enforcement policies may be posted
 separately.
 
 Project maintainers, contributors and users who do not follow or enforce the
-Code of Conduct in good faith may face temporary or permanent repercussions 
+Code of Conduct in good faith may face temporary or permanent repercussions
 with their involvement in the project as determined by the project's leader(s).
 
 ## Attribution

.github/dev-requirements.txt

@@ -0,0 +1,4 @@
+pre-commit
+black==23.3.0
+isort
+flake8

.github/workflows/generate.yaml

@@ -36,7 +36,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
-      matrix:     
+      matrix:
         image: ["ubuntu", "centos", "rockylinux:9.0", "alpine", "busybox"]
 
     name: Generate Matrix
@@ -62,12 +62,12 @@ jobs:
           outfile: ${{ matrix.image }}.json
       - name: View Output
         run: cat ${{ matrix.image }}.json
-        
+
   test-diffs:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
-      matrix:     
+      matrix:
         image: [["vanessa/salad", "vanessa-salad.json"]]
 
     name: Generate Matrix

.github/workflows/main.yml

@@ -21,19 +21,9 @@ jobs:
         with:
           files: ./docs/getting_started/ ./docs/index.rst
 
-      - name: Lint python code
+      - name: Lint and format Python code
         run: |
           export PATH="/usr/share/miniconda/bin:$PATH"
           source activate black
-          pip install black
-          black --check container_guts
-          
-      - name: Check imports with pyflakes
-        run: |
-          export PATH="/usr/share/miniconda/bin:$PATH"
-          source activate black
-          pyflakes container_guts/*.py
-          pyflakes container_guts/client
-          pyflakes container_guts/main/client.py
-          pyflakes container_guts/main/templates.py
-          pyflakes container_guts/main/container/docker.py
+          pip install -r .github/dev-requirements.txt
+          pre-commit run --all-files

.pre-commit-config.yaml

@@ -0,0 +1,25 @@
+exclude: ".all-contributorsrc"
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.3.0
+    hooks:
+      - id: check-added-large-files
+      - id: check-case-conflict
+      - id: check-docstring-first
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+      - id: mixed-line-ending
+
+  - repo: local
+    hooks:
+      - id: black
+        name: black
+        language: python
+        types: [python]
+        entry: black
+
+      - id: flake8
+        name: flake8
+        language: python
+        types: [python]
+        entry: flake8

CHANGELOG.md

@@ -14,10 +14,10 @@ and **Merged pull requests**. Critical items to know are:
 The versions coincide with releases on pip. Only major versions will be released as tags on Github.
 
 ## [0.0.x](https://github.com/singularityhub/guts/tree/main) (0.0.x)
+ - add function for most similar (0.0.16)
  - skip /dev in tar (0.0.15)
  - diff action and support (0.0.14)
  - adding fs (filesystem) extraction support (0.0.13)
  - tag should be own file (0.0.12)
  - Support for output directory (so path prepared by guts) (0.0.11)
  - Initial creation of project (0.0.1)
-

action/diff/action.yaml

@@ -29,11 +29,11 @@ runs:
    - name: Guts Diff for ${{ inputs.image }}
      id: guts_run
      env:
-       image: ${{ inputs.image }} 
-       outfile: ${{ inputs.outfile }} 
-       outdir: ${{ inputs.outdir }} 
-       database: ${{ inputs.database }} 
-     run: | 
+       image: ${{ inputs.image }}
+       outfile: ${{ inputs.outfile }}
+       outdir: ${{ inputs.outdir }}
+       database: ${{ inputs.database }}
+     run: |
         cmd="guts diff"
         if [ "${outfile}" != "" ]; then
             cmd="${cmd} --outfile ${outfile}"
@@ -44,7 +44,7 @@ runs:
         if [ "${database}" != "" ]; then
             cmd="${cmd} --db ${database}"
         fi
-        cmd="${cmd} ${image}" 
+        cmd="${cmd} ${image}"
         printf "${cmd}\n"
         ${cmd}
      shell: bash

action/manifest/action.yaml

@@ -30,11 +30,11 @@ runs:
    - name: Guts for ${{ inputs.image }}
      id: guts_run
      env:
-       image: ${{ inputs.image }} 
-       outfile: ${{ inputs.outfile }} 
-       outdir: ${{ inputs.outdir }} 
+       image: ${{ inputs.image }}
+       outfile: ${{ inputs.outfile }}
+       outdir: ${{ inputs.outdir }}
        includes: ${{ inputs.include }}
-     run: | 
+     run: |
         cmd="guts manifest"
         if [ "${outfile}" != "" ]; then
             cmd="${cmd} --outfile ${outfile}"
@@ -45,7 +45,7 @@ runs:
         for include in ${includes}; do
             cmd="${cmd} --include ${include}"
         done
-        cmd="${cmd} ${image}" 
+        cmd="${cmd} ${image}"
         printf "${cmd}\n"
         ${cmd}
      shell: bash

container_guts/client/__init__.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 __author__ = "Vanessa Sochat"
-__copyright__ = "Copyright 2021-2022, Vanessa Sochat"
+__copyright__ = "Copyright 2021-2024, Vanessa Sochat"
 __license__ = "MPL 2.0"
 
 import argparse
@@ -69,8 +69,20 @@ def get_parser():
         help="Database root (of json files) to use, either filesystem or git URL to clone",
         dest="database",
     )
+    similar = subparsers.add_parser(
+        "similar",
+        description="calculate similarity of your container against a guts database.",
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    for command in [diff, similar]:
+        command.add_argument(
+            "--db",
+            "--database",
+            help="Database root (of json files) to use, either filesystem or git URL to clone",
+            dest="database",
+        )
 
-    for command in manifest, diff:
+    for command in manifest, diff, similar:
         command.add_argument(
             "-i",
             "--include",
@@ -159,6 +171,8 @@ def help(return_code=0):
         from .manifest import main
     elif args.command == "diff":
         from .diff import main
+    elif args.command == "similar":
+        from .similar import main
 
     # Pass on to the correct parser
     return_code = 0

container_guts/client/diff.py

@@ -1,16 +1,16 @@
 __author__ = "Vanessa Sochat"
-__copyright__ = "Copyright 2022, Vanessa Sochat"
+__copyright__ = "Copyright 2022-2024, Vanessa Sochat"
 __license__ = "MPL 2.0"
 
 import json
 import os
 
 import container_guts.utils as utils
+
 from ..main import ManifestGenerator
 
 
 def main(args, parser, extra, subparser):
-
     # Show args to the user
     print("         image: %s" % args.image)
     print("       outfile: %s" % args.outfile)

container_guts/client/manifest.py

@@ -1,16 +1,16 @@
 __author__ = "Vanessa Sochat"
-__copyright__ = "Copyright 2022, Vanessa Sochat"
+__copyright__ = "Copyright 2022-2024, Vanessa Sochat"
 __license__ = "MPL 2.0"
 
 import json
 import os
 
 import container_guts.utils as utils
+
 from ..main import ManifestGenerator
 
 
 def main(args, parser, extra, subparser):
-
     # Show args to the user
     print("         image: %s" % args.image)
     print("       outfile: %s" % args.outfile)

container_guts/client/similar.py

@@ -0,0 +1,41 @@
+__author__ = "Vanessa Sochat"
+__copyright__ = "Copyright 2022-2024, Vanessa Sochat"
+__license__ = "MPL 2.0"
+
+import json
+import os
+
+import container_guts.utils as utils
+
+from ..main import ManifestGenerator
+
+
+def main(args, parser, extra, subparser):
+    # Show args to the user
+    print("         image: %s" % args.image)
+    print("       outfile: %s" % args.outfile)
+    print("        outdir: %s" % args.outdir)
+    print("container tech: %s" % args.container_tech)
+    print("      database: %s" % args.database)
+
+    # Derive an initial manifest
+    cli = ManifestGenerator(tech=args.container_tech)
+    manifests = cli.similar(args.image, database=args.database)
+    outfile = None
+
+    # Default to using outfile first, then outdir if defined
+    if args.outfile:
+        outfile = args.outfile
+    elif args.outdir:
+        outfile = os.path.join(args.outdir, "%s.json" % cli.save_path(args.image))
+        dirname = os.path.dirname(outfile)
+        if not os.path.exists(dirname):
+            os.makedirs(dirname)
+
+    # If we have an output file, make sure to set step output
+    if outfile:
+        print(f"Saving to {outfile}...")
+        print(f"::set-output name=outfile::{outfile}")
+        utils.write_json(manifests, outfile)
+    else:
+        print(json.dumps(manifests, indent=4))

container_guts/defaults.py

@@ -1,10 +1,11 @@
 __author__ = "Vanessa Sochat"
-__copyright__ = "Copyright 2021-2022, Vanessa Sochat"
+__copyright__ = "Copyright 2021-2024, Vanessa Sochat"
 __license__ = "MPL 2.0"
 
-import container_guts.utils as utils
 import os
 
+import container_guts.utils as utils
+
 # Default database for base image
 database = "https://github.com/singularityhub/shpc-guts"
 

container_guts/logger.py

@@ -1,5 +1,5 @@
 __author__ = "Vanessa Sochat"
-__copyright__ = "Copyright 2021-2022, Vanessa Sochat"
+__copyright__ = "Copyright 2021-2024, Vanessa Sochat"
 __license__ = "MPL 2.0"
 
 import inspect
@@ -37,7 +37,6 @@ def add_prefix(msg, char=">>"):
 
 
 class ColorizingStreamHandler(_logging.StreamHandler):
-
     BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(8)
     RESET_SEQ = LogColors.ENDC
     COLOR_SEQ = "\033[%dm"

container_guts/main/client.py

@@ -1,5 +1,5 @@
 __author__ = "Vanessa Sochat"
-__copyright__ = "Copyright 2021-2022, Vanessa Sochat"
+__copyright__ = "Copyright 2021-2024, Vanessa Sochat"
 __license__ = "MPL 2.0"
 
 
@@ -8,10 +8,9 @@
 
 from .. import utils
 from ..logger import logger
-
-from .database import Database
-from .container.decorator import ensure_container
 from .container.base import ContainerName
+from .container.decorator import ensure_container
+from .database import Database
 
 
 class ManifestGenerator:
@@ -60,7 +59,7 @@ def diff(self, image, database=None):
         # Catch the error so we clean up the running container
         try:
             result = db.diff(self.manifests[image.uri])
-        except:
+        except Exception:
             self.container.cleanup(image)
             return
 
@@ -70,6 +69,28 @@ def diff(self, image, database=None):
         self.container.cleanup(image)
         return {image.uri: {"diff": result}}
 
+    @ensure_container
+    def similar(self, image, database=None):
+        """
+        Generate a manifest for an image and similarity scores
+        """
+        print(f"Generating similarity for {image}")
+        tmpdir = self.extract(image, cleanup=False, includes=["paths", "fs"])
+        db = Database(database)
+
+        # Catch the error so we clean up the running container
+        try:
+            result = db.similar(self.manifests[image.uri])
+        except Exception:
+            self.container.cleanup(image)
+            return
+
+        # Only cleans up if was cloned
+        db.cleanup()
+        shutil.rmtree(tmpdir, ignore_errors=True)
+        self.container.cleanup(image)
+        return {image.uri: {"similar": result}}
+
     def extract_filesystem(self, root):
         """
         List all contents of the filesystem
@@ -181,7 +202,6 @@ def get_manifests(self, root):
         for jsonfile in utils.recursive_find(root, "json$"):
             data = utils.read_json(jsonfile)
             if "manifest" in jsonfile:
-
                 continue
             print("Found layer config %s" % jsonfile)