Add sha digest to resolved artifacts (smithy-lang#1961)

Adds a sha digest to resolved artifacts in the resolved artifact cache file and versioning for the cache file. --------- Co-authored-by: Manuel Sugawara <[email protected]>
srchase · Aug 31, 2023 · facbdc3 · facbdc3
1 parent 5944796
commit facbdc3
Show file tree

Hide file tree

Showing 10 changed files with 177 additions and 46 deletions.
diff --git a/smithy-cli/src/it/java/software/amazon/smithy/cli/MavenResolverTest.java b/smithy-cli/src/it/java/software/amazon/smithy/cli/MavenResolverTest.java
@@ -154,8 +154,9 @@ public void invalidatesCacheWhenDependencyChanges() {
             String cacheContents = result.getFile("build", "smithy", "classpath.json");
 
             ObjectNode node = Node.parse(cacheContents).expectObjectNode();
-            String location = node.expectStringMember("software.amazon.smithy:smithy-aws-traits:"
-                                                      + TEST_VERSION).getValue();
+            ObjectNode artifacts = node.expectObjectMember("artifacts");
+            String location = artifacts.expectObjectMember("software.amazon.smithy:smithy-aws-traits:"
+                                                      + TEST_VERSION).expectStringMember("path").getValue();
 
             // Set the lastModified of the JAR to the current time, which is > than the time of the config file,
             // so the cache is invalided.

diff --git a/smithy-cli/src/main/java/software/amazon/smithy/cli/dependencies/FileCacheResolver.java b/smithy-cli/src/main/java/software/amazon/smithy/cli/dependencies/FileCacheResolver.java
@@ -21,7 +21,6 @@
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.time.Duration;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -40,6 +39,7 @@
  * given reference point in time.
  */
 public final class FileCacheResolver implements DependencyResolver {
+    private static final String CURRENT_CACHE_FILE_VERSION = "1.0";
 
     // This is hard-coded for now to 1 day, and it can become an environment variable in the future if needed.
     private static final Duration SMITHY_MAVEN_TTL = Duration.parse("P1D");
@@ -113,17 +113,28 @@ private List<ResolvedArtifact> load() {
             throw new DependencyResolverException("Error loading dependency cache file from " + location, e);
         }
 
-        List<ResolvedArtifact> result = new ArrayList<>(node.getStringMap().size());
-        for (Map.Entry<String, Node> entry : node.getStringMap().entrySet()) {
-            Path artifactLocation = Paths.get(entry.getValue().expectStringNode().getValue());
-            long lastModifiedOfArtifact = artifactLocation.toFile().lastModified();
+        // If the version of the cache file does not match the current version or does not exist
+        // invalidate it so we can replace it with a more recent version.
+        if (!node.containsMember("version")
+                || !CURRENT_CACHE_FILE_VERSION.equals(node.expectStringMember("version").getValue())
+        ) {
+            LOGGER.fine(() -> "Invalidating dependency cache: cache file uses old version");
+            invalidate();
+            return Collections.emptyList();
+        }
+
+        ObjectNode artifactNode = node.expectObjectMember("artifacts");
+        List<ResolvedArtifact> result = new ArrayList<>(artifactNode.getStringMap().size());
+        for (Map.Entry<String, Node> entry : artifactNode.getStringMap().entrySet()) {
+            ResolvedArtifact artifact = ResolvedArtifact.fromNode(entry.getKey(), entry.getValue());
+            long lastModifiedOfArtifact = artifact.getLastModified();
             // Invalidate the cache if the JAR file was updated since the cache was created.
             if (lastModifiedOfArtifact == 0 || lastModifiedOfArtifact > cacheLastModifiedMillis) {
-                LOGGER.fine(() -> "Invalidating dependency cache: artifact is newer than cache: " + artifactLocation);
+                LOGGER.fine(() -> "Invalidating dependency cache: artifact is newer than cache: " + artifact.getPath());
                 invalidate();
                 return Collections.emptyList();
             }
-            result.add(ResolvedArtifact.fromCoordinates(artifactLocation, entry.getKey()));
+            result.add(artifact);
         }
 
         return result;
@@ -139,11 +150,13 @@ private void save(List<ResolvedArtifact> result) {
         try {
             Files.createDirectories(parent);
             ObjectNode.Builder builder = Node.objectNodeBuilder();
+            builder.withMember("version", CURRENT_CACHE_FILE_VERSION);
+            ObjectNode.Builder artifactNodeBuilder = Node.objectNodeBuilder();
             for (ResolvedArtifact artifact : result) {
-                builder.withMember(artifact.getCoordinates(), artifact.getPath().toString());
+                artifactNodeBuilder.withMember(artifact.getCoordinates(), artifact.toNode());
             }
-            ObjectNode objectNode = builder.build();
-            Files.write(filePath, Node.printJson(objectNode).getBytes(StandardCharsets.UTF_8));
+            builder.withMember("artifacts", artifactNodeBuilder.build());
+            Files.write(filePath, Node.printJson(builder.build()).getBytes(StandardCharsets.UTF_8));
         } catch (IOException e) {
             throw new DependencyResolverException("Unable to write classpath cache file: " + e.getMessage(), e);
         }

diff --git a/smithy-cli/src/main/java/software/amazon/smithy/cli/dependencies/ResolvedArtifact.java b/smithy-cli/src/main/java/software/amazon/smithy/cli/dependencies/ResolvedArtifact.java
@@ -16,26 +16,39 @@
 package software.amazon.smithy.cli.dependencies;
 
 import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.Objects;
+import software.amazon.smithy.model.node.Node;
+import software.amazon.smithy.model.node.ObjectNode;
+import software.amazon.smithy.model.node.ToNode;
+import software.amazon.smithy.utils.IoUtils;
 
 /**
  * An artifact resolved from a repository that provides the path on disk where the artifact
  * was downloaded, and the coordinates of the artifact.
  */
-public final class ResolvedArtifact {
+public final class ResolvedArtifact implements ToNode {
+    private static final String SHA_SUM_MEMBER_NAME = "shaSum";
+    private static final String PATH_MEMBER_NAME = "path";
+
     private final Path path;
     private final String coordinates;
     private final String groupId;
     private final String artifactId;
     private final String version;
+    private final String shaSum;
 
     public ResolvedArtifact(Path path, String groupId, String artifactId, String version) {
-        this(path, groupId + ':' + artifactId + ':' + version, groupId, artifactId, version);
+        this(path, groupId + ':' + artifactId + ':' + version, groupId, artifactId,
+                version, IoUtils.computeSha256(path));
     }
 
-    private ResolvedArtifact(Path path, String coordinates, String groupId, String artifactId, String version) {
+    private ResolvedArtifact(Path path, String coordinates, String groupId, String artifactId,
+                             String version, String shaSum
+    ) {
         this.coordinates = Objects.requireNonNull(coordinates);
         this.path = Objects.requireNonNull(path);
+        this.shaSum = Objects.requireNonNull(shaSum);
         this.groupId = groupId;
         this.artifactId = artifactId;
         this.version = version;
@@ -44,17 +57,21 @@ private ResolvedArtifact(Path path, String coordinates, String groupId, String a
     /**
      * Creates a resolved artifact from a file path and Maven coordinates string.
      *
-     * @param location    Location of the artifact.
      * @param coordinates Maven coordinates (e.g., group:artifact:version).
+     * @param node Node containing the resolved artifact data.
      * @return Returns the created artifact.
      * @throws DependencyResolverException if the provided coordinates are invalid.
      */
-    public static ResolvedArtifact fromCoordinates(Path location, String coordinates) {
+    public static ResolvedArtifact fromNode(String coordinates, Node node) {
         String[] parts = coordinates.split(":");
         if (parts.length != 3) {
             throw new DependencyResolverException("Invalid Maven coordinates: " + coordinates);
         }
-        return new ResolvedArtifact(location, coordinates, parts[0], parts[1], parts[2]);
+        ObjectNode objectNode = node.expectObjectNode();
+        Path location = Paths.get(objectNode.expectMember(PATH_MEMBER_NAME).expectStringNode().getValue());
+        String shaSum = objectNode.expectMember(SHA_SUM_MEMBER_NAME).expectStringNode().getValue();
+
+        return new ResolvedArtifact(location, coordinates, parts[0], parts[1], parts[2], shaSum);
     }
 
     /**
@@ -96,14 +113,28 @@ public String getVersion() {
         return version;
     }
 
+    /**
+     * @return Get the sha256 digest of the artifact.
+     */
+    public String getShaSum() {
+        return shaSum;
+    }
+
+    /**
+     * @return Get the last time the artifact was modified.
+     */
+    public long getLastModified() {
+        return path.toFile().lastModified();
+    }
+
     @Override
     public String toString() {
-        return "{path=" + path + ", coordinates='" + coordinates + "'}";
+        return "{path=" + path + ", coordinates='" + coordinates + ", shaSum='" + shaSum + "'}";
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(coordinates, path);
+        return Objects.hash(coordinates, path, shaSum);
     }
 
     @Override
@@ -114,6 +145,16 @@ public boolean equals(Object o) {
             return false;
         }
         ResolvedArtifact artifact = (ResolvedArtifact) o;
-        return path.equals(artifact.path) && coordinates.equals(artifact.coordinates);
+        return path.equals(artifact.path)
+                && shaSum.equals(artifact.shaSum)
+                && coordinates.equals(artifact.coordinates);
+    }
+
+    @Override
+    public Node toNode() {
+        return Node.objectNodeBuilder()
+                .withMember(PATH_MEMBER_NAME, path.toString())
+                .withMember(SHA_SUM_MEMBER_NAME, shaSum)
+                .build();
     }
 }
diff --git a/smithy-cli/src/test/java/software/amazon/smithy/cli/dependencies/FileCacheResolverTest.java b/smithy-cli/src/test/java/software/amazon/smithy/cli/dependencies/FileCacheResolverTest.java
@@ -11,13 +11,15 @@
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
+import java.nio.file.Path;
 import java.time.Duration;
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.List;
 import java.util.function.Consumer;
 import org.junit.jupiter.api.Test;
 import software.amazon.smithy.build.model.MavenRepository;
+import software.amazon.smithy.model.node.Node;
 import software.amazon.smithy.utils.IoUtils;
 import software.amazon.smithy.utils.ListUtils;
 
@@ -41,7 +43,7 @@ public void ignoresAndDeletesEmptyCacheFiles() throws IOException {
         File jar = File.createTempFile("foo", ".json");
 
         List<ResolvedArtifact> result = ListUtils.of(
-                ResolvedArtifact.fromCoordinates(jar.toPath(), "com.foo:bar:1.0.0"));
+                ResolvedArtifact.fromNode("com.foo:bar:1.0.0", getNodeForPath(jar.toPath())));
         Mock mock = new Mock(result);
         DependencyResolver resolver = new FileCacheResolver(cache, System.currentTimeMillis(), mock);
 
@@ -67,7 +69,8 @@ private void validateCacheScenario(Consumer<File> jarFileMutation) throws IOExce
         File jar = File.createTempFile("foo", ".jar");
         Files.write(jar.toPath(), "{}".getBytes(StandardCharsets.UTF_8));
 
-        ResolvedArtifact artifact = ResolvedArtifact.fromCoordinates(jar.toPath(), "com.foo:bar:1.0.0");
+        ResolvedArtifact artifact = ResolvedArtifact.fromNode("com.foo:bar:1.0.0",
+                getNodeForPath(jar.toPath()));
         List<ResolvedArtifact> result = new ArrayList<>();
         result.add(artifact);
 
@@ -104,7 +107,8 @@ public void invalidatesCacheWhenConfigIsNewerThanCache() throws IOException {
         File jar = File.createTempFile("foo", ".jar");
         Files.write(jar.toPath(), "{}".getBytes(StandardCharsets.UTF_8));
 
-        ResolvedArtifact artifact = ResolvedArtifact.fromCoordinates(jar.toPath(), "com.foo:bar:1.0.0");
+        ResolvedArtifact artifact = ResolvedArtifact.fromNode("com.foo:bar:1.0.0",
+                getNodeForPath(jar.toPath()));
         List<ResolvedArtifact> result = new ArrayList<>();
         result.add(artifact);
 
@@ -135,7 +139,8 @@ public void invalidatesCacheWhenCacheExceedsTTL() throws IOException {
         File jar = File.createTempFile("foo", ".jar");
         Files.write(jar.toPath(), "{}".getBytes(StandardCharsets.UTF_8));
 
-        ResolvedArtifact artifact = ResolvedArtifact.fromCoordinates(jar.toPath(), "com.foo:bar:1.0.0");
+        ResolvedArtifact artifact = ResolvedArtifact.fromNode("com.foo:bar:1.0.0",
+                getNodeForPath(jar.toPath()));
         List<ResolvedArtifact> result = new ArrayList<>();
         result.add(artifact);
 
@@ -182,4 +187,11 @@ public List<ResolvedArtifact> resolve() {
             return artifacts;
         }
     }
+
+    private static Node getNodeForPath(Path path) {
+        return Node.objectNodeBuilder()
+                .withMember("path", path.toString())
+                .withMember("shaSum", IoUtils.computeSha256(path))
+                .build();
+    }
 }
diff --git a/...i/src/test/java/software/amazon/smithy/cli/dependencies/FilterCliVersionResolverTest.java b/...i/src/test/java/software/amazon/smithy/cli/dependencies/FilterCliVersionResolverTest.java
@@ -4,13 +4,13 @@
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.empty;
 
-import java.nio.file.Paths;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 import software.amazon.smithy.build.model.MavenRepository;
+import software.amazon.smithy.model.node.Node;
 import software.amazon.smithy.utils.ListUtils;
 
 public class FilterCliVersionResolverTest {
@@ -52,17 +52,17 @@ public void addDependency(String coordinates) {
             @Override
             public List<ResolvedArtifact> resolve() {
                 return Arrays.asList(
-                    ResolvedArtifact.fromCoordinates(Paths.get("/a"), "software.amazon.smithy:smithy-model:1.25.0"),
-                    ResolvedArtifact.fromCoordinates(Paths.get("/b"), "software.amazon.smithy:smithy-utils:1.25.0"),
-                    ResolvedArtifact.fromCoordinates(Paths.get("/c"), "software.amazon.smithy:smithy-other:1.25.0"),
-                    ResolvedArtifact.fromCoordinates(Paths.get("/d"), "software.amazon.foo:foo-other:1.0.0")
+                    ResolvedArtifact.fromNode("software.amazon.smithy:smithy-model:1.25.0", getNodeForPath("/a")),
+                    ResolvedArtifact.fromNode("software.amazon.smithy:smithy-utils:1.25.0", getNodeForPath("/b")),
+                    ResolvedArtifact.fromNode("software.amazon.smithy:smithy-other:1.25.0", getNodeForPath("/c")),
+                    ResolvedArtifact.fromNode("software.amazon.foo:foo-other:1.0.0", getNodeForPath("/d"))
                 );
             }
         });
 
         assertThat(filter.resolve(), contains(
-            ResolvedArtifact.fromCoordinates(Paths.get("/c"), "software.amazon.smithy:smithy-other:1.25.0"),
-            ResolvedArtifact.fromCoordinates(Paths.get("/d"), "software.amazon.foo:foo-other:1.0.0")
+            ResolvedArtifact.fromNode("software.amazon.smithy:smithy-other:1.25.0", getNodeForPath("/c")),
+            ResolvedArtifact.fromNode("software.amazon.foo:foo-other:1.0.0", getNodeForPath("/d"))
         ));
     }
 
@@ -81,11 +81,18 @@ public void addDependency(String coordinates) {
 
             @Override
             public List<ResolvedArtifact> resolve() {
-                return ListUtils.of(ResolvedArtifact.fromCoordinates(Paths.get("/a"),
-                                                                     "software.amazon.smithy:smithy-model:1.27.0"));
+                return ListUtils.of(ResolvedArtifact.fromNode("software.amazon.smithy:smithy-model:1.27.0",
+                        getNodeForPath("/a")));
             }
         });
 
         Assertions.assertThrows(DependencyResolverException.class, filter::resolve);
     }
+
+    private static Node getNodeForPath(String pathStr) {
+        return Node.objectNodeBuilder()
+                .withMember("path", pathStr)
+                .withMember("shaSum", pathStr)
+                .build();
+    }
 }
diff --git a/smithy-cli/src/test/java/software/amazon/smithy/cli/dependencies/ResolvedArtifactTest.java b/smithy-cli/src/test/java/software/amazon/smithy/cli/dependencies/ResolvedArtifactTest.java
@@ -3,28 +3,34 @@
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.equalTo;
 
+import java.net.URISyntaxException;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.Objects;
+
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
+import software.amazon.smithy.model.node.Node;
 
 public class ResolvedArtifactTest {
     @Test
-    public void loadsFromCoordinates() {
-        Path path = Paths.get("/a");
+    public void loadsFromNode() {
         String coordinates = "com.foo:baz-bam:1.2.0";
-        ResolvedArtifact artifact = ResolvedArtifact.fromCoordinates(path, coordinates);
 
-        assertThat(artifact.getPath(), equalTo(path));
+        ResolvedArtifact artifact = ResolvedArtifact.fromNode(coordinates, getNodeForPath());
+
+        assertThat(artifact.getPath(), equalTo(Paths.get("/a")));
         assertThat(artifact.getCoordinates(), equalTo(coordinates));
         assertThat(artifact.getGroupId(), equalTo("com.foo"));
         assertThat(artifact.getArtifactId(), equalTo("baz-bam"));
         assertThat(artifact.getVersion(), equalTo("1.2.0"));
+        assertThat(artifact.getShaSum(), equalTo("sum"));
     }
 
     @Test
-    public void createsCoordinatesStringFromParts() {
-        Path path = Paths.get("/a");
+    public void createsCoordinatesStringFromParts() throws URISyntaxException {
+        Path path =  Paths.get(Objects.requireNonNull(getClass().getResource("test.txt")).toURI());
+
         ResolvedArtifact artifact = new ResolvedArtifact(path, "com.foo", "baz-bam", "1.2.0");
 
         assertThat(artifact.getPath(), equalTo(path));
@@ -36,19 +42,22 @@ public void createsCoordinatesStringFromParts() {
 
     @Test
     public void validatesCoordinatesNotTooManyParts() {
-        Path path = Paths.get("/a");
         String coordinates = "com.foo:baz-bam:1.2.0:boo";
-
         Assertions.assertThrows(DependencyResolverException.class,
-                                () -> ResolvedArtifact.fromCoordinates(path, coordinates));
+                                () -> ResolvedArtifact.fromNode(coordinates, getNodeForPath()));
     }
 
     @Test
     public void validatesCoordinatesEnoughParts() {
-        Path path = Paths.get("/a");
         String coordinates = "com.foo:baz-bam";
-
         Assertions.assertThrows(DependencyResolverException.class,
-                                () -> ResolvedArtifact.fromCoordinates(path, coordinates));
+                                () -> ResolvedArtifact.fromNode(coordinates, getNodeForPath()));
+    }
+
+    private static Node getNodeForPath() {
+        return Node.objectNodeBuilder()
+                .withMember("path", "/a")
+                .withMember("shaSum", "sum")
+                .build();
     }
 }
diff --git a/smithy-cli/src/test/resources/software/amazon/smithy/cli/dependencies/test.txt b/smithy-cli/src/test/resources/software/amazon/smithy/cli/dependencies/test.txt
@@ -0,0 +1 @@
+This is a test.