From a935e9de65e14217d6708c29f7b4b1ce3085be0e Mon Sep 17 00:00:00 2001 From: Josh Basch Date: Thu, 10 Oct 2024 13:12:00 -0700 Subject: [PATCH] Add ExternalReaderRuntime, lots of test coverage --- bench/src/jmh/java/org/pkl/core/ListSort.java | 4 +- .../kotlin/org/pkl/commons/cli/CliCommand.kt | 2 +- .../org/pkl/commons/cli/BaseCommandTest.kt | 31 +++ .../java/org/pkl/core/EvaluatorBuilder.java | 15 +- .../externalProcess/ExternalModuleReader.java | 29 +++ .../ExternalProcessMessagePackEncoder.java | 4 +- .../externalProcess/ExternalReaderBase.java | 30 +++ .../ExternalReaderRuntime.java | 191 ++++++++++++++++++ .../ExternalResourceReader.java | 27 +++ .../pkl/core/messaging/MessageTransport.java | 2 +- .../pkl/core/messaging/MessageTransports.java | 4 +- .../pkl/core/module/ModuleKeyFactories.java | 30 +-- .../java/org/pkl/core/module/ModuleKeys.java | 16 +- .../org/pkl/core/EvaluatorBuilderTest.kt | 10 +- .../externalProcess/MessagePackCodecTest.kt | 73 +++++++ .../TestExternalModuleReader.kt | 38 ++++ .../externalProcess/TestExternalProcess.kt | 130 ++++++++++++ .../TestExternalResourceReader.kt | 31 +++ .../pkl/core/module/ModuleKeyFactoriesTest.kt | 19 ++ .../pkl/core/resource/ResourceReadersTest.kt | 18 ++ .../org/pkl/core/project/project1/PklProject | 18 ++ .../org/pkl/server/AbstractServerTest.kt | 29 +-- 22 files changed, 672 insertions(+), 79 deletions(-) create mode 100644 pkl-core/src/main/java/org/pkl/core/externalProcess/ExternalModuleReader.java create mode 100644 pkl-core/src/main/java/org/pkl/core/externalProcess/ExternalReaderBase.java create mode 100644 pkl-core/src/main/java/org/pkl/core/externalProcess/ExternalReaderRuntime.java create mode 100644 pkl-core/src/main/java/org/pkl/core/externalProcess/ExternalResourceReader.java create mode 100644 pkl-core/src/test/kotlin/org/pkl/core/externalProcess/MessagePackCodecTest.kt create mode 100644 pkl-core/src/test/kotlin/org/pkl/core/externalProcess/TestExternalModuleReader.kt create mode 100644 pkl-core/src/test/kotlin/org/pkl/core/externalProcess/TestExternalProcess.kt create mode 100644 pkl-core/src/test/kotlin/org/pkl/core/externalProcess/TestExternalResourceReader.kt diff --git a/bench/src/jmh/java/org/pkl/core/ListSort.java b/bench/src/jmh/java/org/pkl/core/ListSort.java index 6e22f1a5c..6bc2badd8 100644 --- a/bench/src/jmh/java/org/pkl/core/ListSort.java +++ b/bench/src/jmh/java/org/pkl/core/ListSort.java @@ -28,7 +28,7 @@ import org.pkl.core.repl.ReplRequest; import org.pkl.core.repl.ReplResponse; import org.pkl.core.repl.ReplServer; -import org.pkl.core.resource.ResourceReaderFactories; +import org.pkl.core.resource.ResourceReaders; import org.pkl.core.util.IoUtils; @Warmup(iterations = 5, time = 2) @@ -43,7 +43,7 @@ public class ListSort { HttpClient.dummyClient(), Loggers.stdErr(), List.of(ModuleKeyFactories.standardLibrary), - List.of(ResourceReaderFactories.file()), + List.of(ResourceReaders.file()), Map.of(), Map.of(), null, diff --git a/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliCommand.kt b/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliCommand.kt index 0eb0c16ca..703d8e1b6 100644 --- a/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliCommand.kt +++ b/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliCommand.kt @@ -191,7 +191,7 @@ abstract class CliCommand(protected val cliOptions: CliBaseOptions) { // the same spec // this avoids spawning multiple subprocesses if the same reader implements both reader types // and/or multiple schemes - (externalModuleReaders + externalResourceReaders).values.associateWith { + (externalModuleReaders + externalResourceReaders).values.toSet().associateWith { ExternalProcessImpl(it) } } diff --git a/pkl-commons-cli/src/test/kotlin/org/pkl/commons/cli/BaseCommandTest.kt b/pkl-commons-cli/src/test/kotlin/org/pkl/commons/cli/BaseCommandTest.kt index c538dce6a..0e1e41de9 100644 --- a/pkl-commons-cli/src/test/kotlin/org/pkl/commons/cli/BaseCommandTest.kt +++ b/pkl-commons-cli/src/test/kotlin/org/pkl/commons/cli/BaseCommandTest.kt @@ -24,6 +24,7 @@ import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows import org.pkl.commons.cli.commands.BaseCommand +import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader class BaseCommandTest { @@ -72,4 +73,34 @@ class BaseCommandTest { assertThat(cmd.baseOptions.allowedResources).isEmpty() } + + @Test + fun `--external-resource and --external-module are parsed correctly`() { + cmd.parse( + arrayOf( + "--external-module", + "scheme3=reader3", + "--external-module", + "scheme4=reader4 with args", + "--external-resource", + "scheme1=reader1", + "--external-resource", + "scheme2=reader2 with args" + ) + ) + assertThat(cmd.baseOptions.externalModuleReaders) + .isEqualTo( + mapOf( + "scheme3" to ExternalReader("reader3", emptyList()), + "scheme4" to ExternalReader("reader4", listOf("with", "args")) + ) + ) + assertThat(cmd.baseOptions.externalResourceReaders) + .isEqualTo( + mapOf( + "scheme1" to ExternalReader("reader1", emptyList()), + "scheme2" to ExternalReader("reader2", listOf("with", "args")) + ) + ) + } } diff --git a/pkl-core/src/main/java/org/pkl/core/EvaluatorBuilder.java b/pkl-core/src/main/java/org/pkl/core/EvaluatorBuilder.java index 841a15753..cbe5552f3 100644 --- a/pkl-core/src/main/java/org/pkl/core/EvaluatorBuilder.java +++ b/pkl-core/src/main/java/org/pkl/core/EvaluatorBuilder.java @@ -20,6 +20,8 @@ import java.util.*; import java.util.regex.Pattern; import org.pkl.core.SecurityManagers.StandardBuilder; +import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader; +import org.pkl.core.externalProcess.ExternalProcess; import org.pkl.core.externalProcess.ExternalProcessImpl; import org.pkl.core.http.HttpClient; import org.pkl.core.module.ModuleKeyFactories; @@ -475,16 +477,21 @@ public EvaluatorBuilder applyFromProject(Project project) { } else if (settings.moduleCacheDir() != null) { setModuleCacheDir(settings.moduleCacheDir()); } + + // this isn't ideal as project and non-project ExternalProcessImpl instances can be dupes + var procs = new HashMap(); if (settings.externalModuleReaders() != null) { for (var entry : settings.externalModuleReaders().entrySet()) { - var process = new ExternalProcessImpl(entry.getValue()); - addModuleKeyFactory(ModuleKeyFactories.external(entry.getKey(), process)); + addModuleKeyFactory( + ModuleKeyFactories.external( + entry.getKey(), procs.computeIfAbsent(entry.getValue(), ExternalProcessImpl::new))); } } if (settings.externalResourceReaders() != null) { for (var entry : settings.externalResourceReaders().entrySet()) { - var process = new ExternalProcessImpl(entry.getValue()); - addResourceReader(ResourceReaders.external(entry.getKey(), process)); + addResourceReader( + ResourceReaders.external( + entry.getKey(), procs.computeIfAbsent(entry.getValue(), ExternalProcessImpl::new))); } } return this; diff --git a/pkl-core/src/main/java/org/pkl/core/externalProcess/ExternalModuleReader.java b/pkl-core/src/main/java/org/pkl/core/externalProcess/ExternalModuleReader.java new file mode 100644 index 000000000..e5aca409b --- /dev/null +++ b/pkl-core/src/main/java/org/pkl/core/externalProcess/ExternalModuleReader.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.core.externalProcess; + +import java.net.URI; +import org.pkl.core.messaging.Messages.ModuleReaderSpec; + +public interface ExternalModuleReader extends ExternalReaderBase { + boolean isLocal(); + + String read(URI uri) throws Exception; + + default ModuleReaderSpec getSpec() { + return new ModuleReaderSpec(getScheme(), hasHierarchicalUris(), isLocal(), isGlobbable()); + } +} diff --git a/pkl-core/src/main/java/org/pkl/core/externalProcess/ExternalProcessMessagePackEncoder.java b/pkl-core/src/main/java/org/pkl/core/externalProcess/ExternalProcessMessagePackEncoder.java index 2a30c073a..c6f9e3e49 100644 --- a/pkl-core/src/main/java/org/pkl/core/externalProcess/ExternalProcessMessagePackEncoder.java +++ b/pkl-core/src/main/java/org/pkl/core/externalProcess/ExternalProcessMessagePackEncoder.java @@ -52,7 +52,7 @@ public ExternalProcessMessagePackEncoder(OutputStream outputStream) { } case INITIALIZE_MODULE_READER_RESPONSE -> { var m = (InitializeModuleReaderResponse) msg; - packer.packMapHeader(2); + packMapHeader(1, m.getSpec()); packKeyValue("requestId", m.getRequestId()); if (m.getSpec() != null) { packer.packString("spec"); @@ -61,7 +61,7 @@ public ExternalProcessMessagePackEncoder(OutputStream outputStream) { } case INITIALIZE_RESOURCE_READER_RESPONSE -> { var m = (InitializeResourceReaderResponse) msg; - packer.packMapHeader(2); + packMapHeader(1, m.getSpec()); packKeyValue("requestId", m.getRequestId()); if (m.getSpec() != null) { packer.packString("spec"); diff --git a/pkl-core/src/main/java/org/pkl/core/externalProcess/ExternalReaderBase.java b/pkl-core/src/main/java/org/pkl/core/externalProcess/ExternalReaderBase.java new file mode 100644 index 000000000..342af9e3c --- /dev/null +++ b/pkl-core/src/main/java/org/pkl/core/externalProcess/ExternalReaderBase.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.core.externalProcess; + +import java.net.URI; +import java.util.List; +import org.pkl.core.module.PathElement; + +public interface ExternalReaderBase { + String getScheme(); + + boolean hasHierarchicalUris(); + + boolean isGlobbable(); + + List listElements(URI uri) throws Exception; +} diff --git a/pkl-core/src/main/java/org/pkl/core/externalProcess/ExternalReaderRuntime.java b/pkl-core/src/main/java/org/pkl/core/externalProcess/ExternalReaderRuntime.java new file mode 100644 index 000000000..58b717256 --- /dev/null +++ b/pkl-core/src/main/java/org/pkl/core/externalProcess/ExternalReaderRuntime.java @@ -0,0 +1,191 @@ +/** + * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.core.externalProcess; + +import java.io.IOException; +import java.util.List; +import org.pkl.core.externalProcess.ExternalProcessMessages.*; +import org.pkl.core.messaging.Message.Type; +import org.pkl.core.messaging.MessageTransport; +import org.pkl.core.messaging.Messages.*; +import org.pkl.core.messaging.ProtocolException; +import org.pkl.core.util.Nullable; + +public class ExternalReaderRuntime { + + private final List moduleReaders; + private final List resourceReaders; + private final MessageTransport transport; + + public ExternalReaderRuntime( + List moduleReaders, + List resourceReaders, + MessageTransport transport) { + this.moduleReaders = moduleReaders; + this.resourceReaders = resourceReaders; + this.transport = transport; + } + + public void close() { + transport.close(); + } + + private @Nullable ExternalModuleReader findModuleReader(String scheme) { + for (var moduleReader : moduleReaders) { + if (moduleReader.getScheme().equalsIgnoreCase(scheme)) { + return moduleReader; + } + } + return null; + } + + private @Nullable ExternalResourceReader findResourceReader(String scheme) { + for (var resourceReader : resourceReaders) { + if (resourceReader.getScheme().equalsIgnoreCase(scheme)) { + return resourceReader; + } + } + return null; + } + + public void run() throws ProtocolException, IOException { + transport.start( + (msg) -> { + if (msg.getType() == Type.CLOSE_EXTERNAL_PROCESS) { + close(); + } else { + throw new ProtocolException("Unexpected incoming one-way message: " + msg); + } + }, + (msg) -> { + switch (msg.getType()) { + case INITIALIZE_MODULE_READER_REQUEST -> { + var req = (InitializeModuleReaderRequest) msg; + var reader = findModuleReader(req.getScheme()); + @Nullable ModuleReaderSpec spec = null; + if (reader != null) { + spec = reader.getSpec(); + } + transport.send(new InitializeModuleReaderResponse(req.getRequestId(), spec)); + } + case INITIALIZE_RESOURCE_READER_REQUEST -> { + var req = (InitializeResourceReaderRequest) msg; + var reader = findResourceReader(req.getScheme()); + @Nullable ResourceReaderSpec spec = null; + if (reader != null) { + spec = reader.getSpec(); + } + transport.send(new InitializeResourceReaderResponse(req.getRequestId(), spec)); + } + case LIST_MODULES_REQUEST -> { + var req = (ListModulesRequest) msg; + var reader = findModuleReader(req.getUri().getScheme()); + if (reader == null) { + transport.send( + new ListModulesResponse( + req.getRequestId(), + req.getEvaluatorId(), + null, + "No module reader found for scheme " + req.getUri().getScheme())); + return; + } + try { + transport.send( + new ListModulesResponse( + req.getRequestId(), + req.getEvaluatorId(), + reader.listElements(req.getUri()), + null)); + } catch (Exception e) { + transport.send( + new ListModulesResponse( + req.getRequestId(), req.getEvaluatorId(), null, e.toString())); + } + } + case LIST_RESOURCES_REQUEST -> { + var req = (ListResourcesRequest) msg; + var reader = findModuleReader(req.getUri().getScheme()); + if (reader == null) { + transport.send( + new ListResourcesResponse( + req.getRequestId(), + req.getEvaluatorId(), + null, + "No resource reader found for scheme " + req.getUri().getScheme())); + return; + } + try { + transport.send( + new ListResourcesResponse( + req.getRequestId(), + req.getEvaluatorId(), + reader.listElements(req.getUri()), + null)); + } catch (Exception e) { + transport.send( + new ListResourcesResponse( + req.getRequestId(), req.getEvaluatorId(), null, e.toString())); + } + } + case READ_MODULE_REQUEST -> { + var req = (ReadModuleRequest) msg; + var reader = findModuleReader(req.getUri().getScheme()); + if (reader == null) { + transport.send( + new ReadModuleResponse( + req.getRequestId(), + req.getEvaluatorId(), + null, + "No module reader found for scheme " + req.getUri().getScheme())); + return; + } + try { + transport.send( + new ReadModuleResponse( + req.getRequestId(), req.getEvaluatorId(), reader.read(req.getUri()), null)); + } catch (Exception e) { + transport.send( + new ReadModuleResponse( + req.getRequestId(), req.getEvaluatorId(), null, e.toString())); + } + } + case READ_RESOURCE_REQUEST -> { + var req = (ReadResourceRequest) msg; + var reader = findResourceReader(req.getUri().getScheme()); + if (reader == null) { + transport.send( + new ReadResourceResponse( + req.getRequestId(), + req.getEvaluatorId(), + null, + "No resource reader found for scheme " + req.getUri().getScheme())); + return; + } + try { + transport.send( + new ReadResourceResponse( + req.getRequestId(), req.getEvaluatorId(), reader.read(req.getUri()), null)); + } catch (Exception e) { + transport.send( + new ReadResourceResponse( + req.getRequestId(), req.getEvaluatorId(), new byte[0], e.toString())); + } + } + default -> throw new ProtocolException("Unexpected incoming request message: " + msg); + } + }); + } +} diff --git a/pkl-core/src/main/java/org/pkl/core/externalProcess/ExternalResourceReader.java b/pkl-core/src/main/java/org/pkl/core/externalProcess/ExternalResourceReader.java new file mode 100644 index 000000000..f0fe7e17c --- /dev/null +++ b/pkl-core/src/main/java/org/pkl/core/externalProcess/ExternalResourceReader.java @@ -0,0 +1,27 @@ +/** + * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.core.externalProcess; + +import java.net.URI; +import org.pkl.core.messaging.Messages.ResourceReaderSpec; + +public interface ExternalResourceReader extends ExternalReaderBase { + byte[] read(URI uri) throws Exception; + + default ResourceReaderSpec getSpec() { + return new ResourceReaderSpec(getScheme(), hasHierarchicalUris(), isGlobbable()); + } +} diff --git a/pkl-core/src/main/java/org/pkl/core/messaging/MessageTransport.java b/pkl-core/src/main/java/org/pkl/core/messaging/MessageTransport.java index cb1ccc7fd..fa11925ef 100644 --- a/pkl-core/src/main/java/org/pkl/core/messaging/MessageTransport.java +++ b/pkl-core/src/main/java/org/pkl/core/messaging/MessageTransport.java @@ -24,7 +24,7 @@ interface OneWayHandler { } interface RequestHandler { - void handleRequest(Message.Request msg) throws ProtocolException; + void handleRequest(Message.Request msg) throws ProtocolException, IOException; } interface ResponseHandler { diff --git a/pkl-core/src/main/java/org/pkl/core/messaging/MessageTransports.java b/pkl-core/src/main/java/org/pkl/core/messaging/MessageTransports.java index cba393e45..1be559c5e 100644 --- a/pkl-core/src/main/java/org/pkl/core/messaging/MessageTransports.java +++ b/pkl-core/src/main/java/org/pkl/core/messaging/MessageTransports.java @@ -110,7 +110,7 @@ protected void doStart() {} protected void doClose() {} @Override - protected void doSend(Message message) throws ProtocolException { + protected void doSend(Message message) throws ProtocolException, IOException { other.accept(message); } @@ -142,7 +142,7 @@ protected void log(String message, Object... args) { protected abstract void doSend(Message message) throws ProtocolException, IOException; - protected void accept(Message message) throws ProtocolException { + protected void accept(Message message) throws ProtocolException, IOException { log("Received message: {0}", message); if (message instanceof Message.OneWay msg) { oneWayHandler.handleOneWay(msg); diff --git a/pkl-core/src/main/java/org/pkl/core/module/ModuleKeyFactories.java b/pkl-core/src/main/java/org/pkl/core/module/ModuleKeyFactories.java index b79464d13..139a181fc 100644 --- a/pkl-core/src/main/java/org/pkl/core/module/ModuleKeyFactories.java +++ b/pkl-core/src/main/java/org/pkl/core/module/ModuleKeyFactories.java @@ -34,7 +34,6 @@ /** Utilities for obtaining and using module key factories. */ public final class ModuleKeyFactories { - private ModuleKeyFactories() {} /** A factory for standard library module keys. */ @@ -104,20 +103,16 @@ public static void closeQuietly(Iterable factories) { } private static class StandardLibrary implements ModuleKeyFactory { - private StandardLibrary() {} @Override public Optional create(URI uri) { - if (!uri.getScheme().equals("pkl")) { - return Optional.empty(); - } + if (!uri.getScheme().equals("pkl")) return Optional.empty(); return Optional.of(ModuleKeys.standardLibrary(uri)); } } private static class ModulePath implements ModuleKeyFactory { - final ModulePathResolver resolver; public ModulePath(ModulePathResolver resolver) { @@ -150,7 +145,6 @@ public void close() { } private static class ClassPath implements ModuleKeyFactory { - private final ClassLoader classLoader; public ClassPath(ClassLoader classLoader) { @@ -159,15 +153,12 @@ public ClassPath(ClassLoader classLoader) { @Override public Optional create(URI uri) { - if (!uri.getScheme().equals("modulepath")) { - return Optional.empty(); - } + if (!uri.getScheme().equals("modulepath")) return Optional.empty(); return Optional.of(ModuleKeys.classPath(uri, classLoader)); } } private static class File implements ModuleKeyFactory { - @Override public Optional create(URI uri) { // skip loading providers if the scheme is `file`. @@ -188,7 +179,6 @@ public Optional create(URI uri) { } private static class Http implements ModuleKeyFactory { - private Http() {} @Override @@ -202,17 +192,12 @@ public Optional create(URI uri) { } private static class GenericUrl implements ModuleKeyFactory { - private GenericUrl() {} @Override public Optional create(URI uri) { - if (!uri.isAbsolute()) { - return Optional.empty(); - } - if (uri.isOpaque() && !"jar".equalsIgnoreCase(uri.getScheme())) { - return Optional.empty(); - } + if (!uri.isAbsolute()) return Optional.empty(); + if (uri.isOpaque() && !"jar".equalsIgnoreCase(uri.getScheme())) return Optional.empty(); // Blindly accept this URI, assuming ModuleKeys.genericUrl() can handle it. // This means that ModuleKeyFactories.GenericUrl must come last in the handler chain. @@ -227,7 +212,6 @@ public Optional create(URI uri) { * optionally, a local project declared as a dependency of the current project. */ private static final class Package implements ModuleKeyFactory { - public Optional create(URI uri) throws URISyntaxException { if (uri.getScheme().equalsIgnoreCase("package")) { return Optional.of(ModuleKeys.pkg(uri)); @@ -243,7 +227,6 @@ public Optional create(URI uri) throws URISyntaxException { * dependency, or a local dependency */ private static final class ProjectPackage implements ModuleKeyFactory { - public Optional create(URI uri) throws URISyntaxException { if (uri.getScheme().equalsIgnoreCase("projectpackage")) { return Optional.of(ModuleKeys.projectpackage(uri)); @@ -253,7 +236,6 @@ public Optional create(URI uri) throws URISyntaxException { } private static class FromServiceProviders { - private static final List INSTANCE; static { @@ -288,9 +270,7 @@ private ModuleKeys.External.Resolver getResolver() throws ExternalProcessExcepti public Optional create(URI uri) throws URISyntaxException, ExternalProcessException, IOException { - if (!uri.getScheme().equalsIgnoreCase(scheme)) { - return Optional.empty(); - } + if (!scheme.equalsIgnoreCase(uri.getScheme())) return Optional.empty(); var spec = process.getModuleReaderSpec(scheme); if (spec == null) { diff --git a/pkl-core/src/main/java/org/pkl/core/module/ModuleKeys.java b/pkl-core/src/main/java/org/pkl/core/module/ModuleKeys.java index db9809f8c..780edee26 100644 --- a/pkl-core/src/main/java/org/pkl/core/module/ModuleKeys.java +++ b/pkl-core/src/main/java/org/pkl/core/module/ModuleKeys.java @@ -53,7 +53,6 @@ /** Utilities for creating and using {@link ModuleKey}s. */ public final class ModuleKeys { - private ModuleKeys() {} /** @@ -154,7 +153,6 @@ public static ModuleKey cached(ModuleKey delegate, String text) { } private static class CachedModuleKey implements ModuleKey, ResolvedModuleKey { - private final ModuleKey delegate; private final String text; @@ -212,7 +210,6 @@ public List listElements(SecurityManager securityManager, URI baseU } private static class Synthetic implements ModuleKey { - final URI uri; final URI importBaseUri; final boolean isCached; @@ -259,7 +256,6 @@ public boolean isCached() { } private static class StandardLibrary implements ModuleKey, ResolvedModuleKey { - final URI uri; StandardLibrary(URI uri) { @@ -309,7 +305,6 @@ public String loadSource() throws IOException { } private static class File implements ModuleKey { - final URI uri; File(URI uri) { @@ -368,7 +363,6 @@ public boolean hasHierarchicalUris() { } private static final class ModulePath implements ModuleKey { - final URI uri; final ModulePathResolver resolver; @@ -420,7 +414,6 @@ public ResolvedModuleKey resolve(SecurityManager securityManager) } private static final class ClassPath implements ModuleKey { - final URI uri; final ClassLoader classLoader; @@ -467,9 +460,7 @@ public ResolvedModuleKey resolve(SecurityManager securityManager) throws IOException, SecurityManagerException { securityManager.checkResolveModule(uri); var url = classLoader.getResource(getResourcePath()); - if (url == null) { - throw new FileNotFoundException(); - } + if (url == null) throw new FileNotFoundException(); try { return ResolvedModuleKeys.url(this, url.toURI(), url); } catch (URISyntaxException e) { @@ -485,7 +476,6 @@ private String getResourcePath() { } private static class Http implements ModuleKey { - private final URI uri; Http(URI uri) { @@ -524,7 +514,6 @@ public ResolvedModuleKey resolve(SecurityManager securityManager) } private static class GenericUrl implements ModuleKey { - final URI uri; GenericUrl(URI uri) { @@ -576,7 +565,6 @@ public ResolvedModuleKey resolve(SecurityManager securityManager) } private abstract static class AbstractPackage implements ModuleKey { - protected final PackageAssetUri packageAssetUri; AbstractPackage(PackageAssetUri packageAssetUri) { @@ -798,9 +786,7 @@ public boolean hasElement(SecurityManager securityManager, URI elementUri) } public static class External implements ModuleKey { - public static class Resolver { - private final MessageTransport transport; private final long evaluatorId; private final Map> readResponses = new ConcurrentHashMap<>(); diff --git a/pkl-core/src/test/kotlin/org/pkl/core/EvaluatorBuilderTest.kt b/pkl-core/src/test/kotlin/org/pkl/core/EvaluatorBuilderTest.kt index 77b3c9daa..135ba3ed8 100644 --- a/pkl-core/src/test/kotlin/org/pkl/core/EvaluatorBuilderTest.kt +++ b/pkl-core/src/test/kotlin/org/pkl/core/EvaluatorBuilderTest.kt @@ -71,8 +71,10 @@ class EvaluatorBuilderTest { fun `sets evaluator settings from project`() { val projectPath = Path.of(javaClass.getResource("project/project1/PklProject")!!.toURI()) val project = Project.loadFromPath(projectPath, SecurityManagers.defaultManager, null) - val projectDir = Path.of(javaClass.getResource("project/project1/PklProject")!!.toURI()).parent - val builder = EvaluatorBuilder.unconfigured().applyFromProject(project) + val projectDir = projectPath.parent + val builder = EvaluatorBuilder.unconfigured() + val moduleKeyFactoryCount = builder.moduleKeyFactories.size + builder.applyFromProject(project) assertThat(builder.allowedResources.map { it.pattern() }).isEqualTo(listOf("foo:", "bar:")) assertThat(builder.allowedModules.map { it.pattern() }).isEqualTo(listOf("baz:", "biz:")) assertThat(builder.externalProperties).isEqualTo(mapOf("one" to "1")) @@ -80,5 +82,9 @@ class EvaluatorBuilderTest { assertThat(builder.moduleCacheDir).isEqualTo(projectDir.resolve("my-cache-dir/")) assertThat(builder.rootDir).isEqualTo(projectDir.resolve("my-root-dir/")) assertThat(builder.timeout).isEqualTo(Duration.ofMinutes(5L)) + assertThat(builder.moduleKeyFactories.size - moduleKeyFactoryCount) + .isEqualTo(3) // two external readers, one module path + assertThat(builder.resourceReaders.find { it.uriScheme == "scheme3" }).isNotNull + assertThat(builder.resourceReaders.find { it.uriScheme == "scheme4" }).isNotNull } } diff --git a/pkl-core/src/test/kotlin/org/pkl/core/externalProcess/MessagePackCodecTest.kt b/pkl-core/src/test/kotlin/org/pkl/core/externalProcess/MessagePackCodecTest.kt new file mode 100644 index 000000000..3d8852adf --- /dev/null +++ b/pkl-core/src/test/kotlin/org/pkl/core/externalProcess/MessagePackCodecTest.kt @@ -0,0 +1,73 @@ +/** + * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.core.externalProcess + +import java.io.PipedInputStream +import java.io.PipedOutputStream +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import org.msgpack.core.MessagePack +import org.pkl.core.externalProcess.ExternalProcessMessages.* +import org.pkl.core.messaging.* + +class MessagePackCodecTest { + private val encoder: MessageEncoder + private val decoder: MessageDecoder + + init { + val inputStream = PipedInputStream() + val outputStream = PipedOutputStream(inputStream) + encoder = ExternalProcessMessagePackEncoder(MessagePack.newDefaultPacker(outputStream)) + decoder = ExternalProcessMessagePackDecoder(MessagePack.newDefaultUnpacker(inputStream)) + } + + private fun roundtrip(message: Message) { + encoder.encode(message) + val decoded = decoder.decode() + assertThat(decoded).isEqualTo(message) + } + + @Test + fun `round-trip InitializeModuleReaderRequest`() { + roundtrip(InitializeModuleReaderRequest(123, "my-scheme")) + } + + @Test + fun `round-trip InitializeResourceReaderRequest`() { + roundtrip(InitializeResourceReaderRequest(123, "my-scheme")) + } + + @Test + fun `round-trip InitializeModuleReaderResponse`() { + roundtrip(InitializeModuleReaderResponse(123, null)) + roundtrip( + InitializeModuleReaderResponse(123, Messages.ModuleReaderSpec("my-scheme", true, true, true)) + ) + } + + @Test + fun `round-trip InitializeResourceReaderResponse`() { + roundtrip(InitializeResourceReaderResponse(123, null)) + roundtrip( + InitializeResourceReaderResponse(123, Messages.ResourceReaderSpec("my-scheme", true, true)) + ) + } + + @Test + fun `round-trip CloseExternalProcess`() { + roundtrip(CloseExternalProcess()) + } +} diff --git a/pkl-core/src/test/kotlin/org/pkl/core/externalProcess/TestExternalModuleReader.kt b/pkl-core/src/test/kotlin/org/pkl/core/externalProcess/TestExternalModuleReader.kt new file mode 100644 index 000000000..dc1c5f265 --- /dev/null +++ b/pkl-core/src/test/kotlin/org/pkl/core/externalProcess/TestExternalModuleReader.kt @@ -0,0 +1,38 @@ +/** + * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.core.externalProcess + +import java.net.URI +import org.pkl.core.module.PathElement + +class TestExternalModuleReader : ExternalModuleReader { + override fun getScheme(): String = "test" + + override fun hasHierarchicalUris(): Boolean = false + + override fun isLocal(): Boolean = true + + override fun isGlobbable(): Boolean = false + + override fun read(uri: URI): String = + """ + name = "Pigeon" + age = 40 + """ + .trimIndent() + + override fun listElements(uri: URI): List = emptyList() +} diff --git a/pkl-core/src/test/kotlin/org/pkl/core/externalProcess/TestExternalProcess.kt b/pkl-core/src/test/kotlin/org/pkl/core/externalProcess/TestExternalProcess.kt new file mode 100644 index 000000000..d6082c007 --- /dev/null +++ b/pkl-core/src/test/kotlin/org/pkl/core/externalProcess/TestExternalProcess.kt @@ -0,0 +1,130 @@ +/** + * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.core.externalProcess + +import java.io.IOException +import java.io.PipedInputStream +import java.io.PipedOutputStream +import java.util.concurrent.CompletableFuture +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.ExecutionException +import java.util.concurrent.Future +import kotlin.random.Random +import org.pkl.core.externalProcess.ExternalProcessMessages.* +import org.pkl.core.messaging.MessageTransport +import org.pkl.core.messaging.MessageTransports +import org.pkl.core.messaging.Messages.* +import org.pkl.core.messaging.ProtocolException + +class TestExternalProcess(private val transport: MessageTransport) : ExternalProcess { + private val initializeModuleReaderResponses: MutableMap> = + ConcurrentHashMap() + private val initializeResourceReaderResponses: MutableMap> = + ConcurrentHashMap() + + override fun close() { + transport.send(CloseExternalProcess()) + transport.close() + } + + override fun getTransport(): MessageTransport = transport + + fun run() { + try { + transport.start( + { throw ProtocolException("Unexpected incoming one-way message: $it") }, + { throw ProtocolException("Unexpected incoming request message: $it") }, + ) + } catch (e: ProtocolException) { + throw RuntimeException(e) + } catch (e: IOException) { + throw RuntimeException(e) + } + } + + override fun getModuleReaderSpec(scheme: String): ModuleReaderSpec? = + initializeModuleReaderResponses + .computeIfAbsent(scheme) { + CompletableFuture().apply { + val request = InitializeModuleReaderRequest(Random.nextLong(), scheme) + transport.send(request) { response -> + when (response) { + is InitializeModuleReaderResponse -> { + complete(response.spec) + } + else -> completeExceptionally(ProtocolException("unexpected response")) + } + } + } + } + .getUnderlying() + + override fun getResourceReaderSpec(scheme: String): ResourceReaderSpec? = + initializeResourceReaderResponses + .computeIfAbsent(scheme) { + CompletableFuture().apply { + val request = InitializeResourceReaderRequest(Random.nextLong(), scheme) + transport.send(request) { response -> + when (response) { + is InitializeResourceReaderResponse -> { + complete(response.spec) + } + else -> completeExceptionally(ProtocolException("unexpected response")) + } + } + } + } + .getUnderlying() + + companion object { + fun initializeTestHarness( + moduleReaders: List, + resourceReaders: List + ): Pair { + val rxIn = PipedInputStream(10240) + val rxOut = PipedOutputStream(rxIn) + val txIn = PipedInputStream(10240) + val txOut = PipedOutputStream(txIn) + val serverTransport = + MessageTransports.stream( + ExternalProcessMessagePackDecoder(rxIn), + ExternalProcessMessagePackEncoder(txOut), + {} + ) + val clientTransport = + MessageTransports.stream( + ExternalProcessMessagePackDecoder(txIn), + ExternalProcessMessagePackEncoder(rxOut), + {} + ) + + val runtime = ExternalReaderRuntime(moduleReaders, resourceReaders, clientTransport) + val proc = TestExternalProcess(serverTransport) + + Thread(runtime::run).start() + Thread(proc::run).start() + + return proc to runtime + } + } +} + +fun Future.getUnderlying(): T = + try { + get() + } catch (e: ExecutionException) { + throw e.cause!! + } diff --git a/pkl-core/src/test/kotlin/org/pkl/core/externalProcess/TestExternalResourceReader.kt b/pkl-core/src/test/kotlin/org/pkl/core/externalProcess/TestExternalResourceReader.kt new file mode 100644 index 000000000..c77d9cfb7 --- /dev/null +++ b/pkl-core/src/test/kotlin/org/pkl/core/externalProcess/TestExternalResourceReader.kt @@ -0,0 +1,31 @@ +/** + * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.core.externalProcess + +import java.net.URI +import org.pkl.core.module.PathElement + +class TestExternalResourceReader : ExternalResourceReader { + override fun getScheme(): String = "test" + + override fun hasHierarchicalUris(): Boolean = false + + override fun isGlobbable(): Boolean = false + + override fun read(uri: URI): ByteArray = "success".toByteArray(Charsets.UTF_8) + + override fun listElements(uri: URI): List = emptyList() +} diff --git a/pkl-core/src/test/kotlin/org/pkl/core/module/ModuleKeyFactoriesTest.kt b/pkl-core/src/test/kotlin/org/pkl/core/module/ModuleKeyFactoriesTest.kt index 169bd6913..32a0120bf 100644 --- a/pkl-core/src/test/kotlin/org/pkl/core/module/ModuleKeyFactoriesTest.kt +++ b/pkl-core/src/test/kotlin/org/pkl/core/module/ModuleKeyFactoriesTest.kt @@ -26,6 +26,7 @@ import org.pkl.commons.createParentDirectories import org.pkl.commons.toPath import org.pkl.commons.writeString import org.pkl.core.SecurityManagers +import org.pkl.core.externalProcess.* class ModuleKeyFactoriesTest { @Test @@ -126,4 +127,22 @@ class ModuleKeyFactoriesTest { val module2 = factory.create(URI("other")) assertThat(module2).isNotPresent } + + @Test + fun external() { + val extReader = TestExternalModuleReader() + val (proc, runtime) = TestExternalProcess.initializeTestHarness(listOf(extReader), emptyList()) + + val factory = ModuleKeyFactories.external(extReader.scheme, proc) + + val module = factory.create(URI("test:foo")) + assertThat(module).isPresent + assertThat(module.get().uri.scheme).isEqualTo("test") + + val module2 = factory.create(URI("other")) + assertThat(module2).isNotPresent + + proc.close() + runtime.close() + } } diff --git a/pkl-core/src/test/kotlin/org/pkl/core/resource/ResourceReadersTest.kt b/pkl-core/src/test/kotlin/org/pkl/core/resource/ResourceReadersTest.kt index 345864294..a7e36988b 100644 --- a/pkl-core/src/test/kotlin/org/pkl/core/resource/ResourceReadersTest.kt +++ b/pkl-core/src/test/kotlin/org/pkl/core/resource/ResourceReadersTest.kt @@ -23,6 +23,8 @@ import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows import org.junit.jupiter.api.io.TempDir +import org.pkl.core.externalProcess.TestExternalProcess +import org.pkl.core.externalProcess.TestExternalResourceReader import org.pkl.core.module.ModulePathResolver class ResourceReadersTest { @@ -132,4 +134,20 @@ class ResourceReadersTest { assertThat(resource).contains("success") } + + @Test + fun external() { + val extReader = TestExternalResourceReader() + val (proc, runtime) = TestExternalProcess.initializeTestHarness(emptyList(), listOf(extReader)) + + val reader = ResourceReaders.external(extReader.scheme, proc) + val resource = reader.read(URI("test:foo")) + + assertThat(resource).isPresent + assertThat(resource.get()).isInstanceOf(Resource::class.java) + assertThat((resource.get() as Resource).text).contains("success") + + proc.close() + runtime.close() + } } diff --git a/pkl-core/src/test/resources/org/pkl/core/project/project1/PklProject b/pkl-core/src/test/resources/org/pkl/core/project/project1/PklProject index 266ba3da7..35c76e480 100644 --- a/pkl-core/src/test/resources/org/pkl/core/project/project1/PklProject +++ b/pkl-core/src/test/resources/org/pkl/core/project/project1/PklProject @@ -22,4 +22,22 @@ evaluatorSettings { noCache = false rootDir = "my-root-dir/" timeout = 5.min + externalModuleReaders { + ["scheme1"] { + executable = "reader1" + } + ["scheme2"] { + executable = "reader2" + arguments { "with"; "args" } + } + } + externalResourceReaders { + ["scheme3"] { + executable = "reader3" + } + ["scheme4"] { + executable = "reader4" + arguments { "with"; "args" } + } + } } diff --git a/pkl-server/src/test/kotlin/org/pkl/server/AbstractServerTest.kt b/pkl-server/src/test/kotlin/org/pkl/server/AbstractServerTest.kt index 6fdb11bfa..15ff6bda4 100644 --- a/pkl-server/src/test/kotlin/org/pkl/server/AbstractServerTest.kt +++ b/pkl-server/src/test/kotlin/org/pkl/server/AbstractServerTest.kt @@ -179,8 +179,7 @@ abstract class AbstractServerTest { @Test fun `read resource -- null contents and null error`() { - val reader = - ResourceReaderSpec(scheme = "bahumbug", hasHierarchicalUris = true, isGlobbable = false) + val reader = ResourceReaderSpec("bahumbug", true, false) val evaluatorId = client.sendCreateEvaluatorRequest(resourceReaders = listOf(reader)) client.send( @@ -197,14 +196,7 @@ abstract class AbstractServerTest { assertThat(readResourceMsg.uri.toString()).isEqualTo("bahumbug:/foo.pkl") assertThat(readResourceMsg.evaluatorId).isEqualTo(evaluatorId) - client.send( - ReadResourceResponse( - requestId = readResourceMsg.requestId, - evaluatorId = evaluatorId, - contents = null, - error = null - ) - ) + client.send(ReadResourceResponse(readResourceMsg.requestId, evaluatorId, ByteArray(0), null)) val evaluateResponse = client.receive() assertThat(evaluateResponse.error).isNull() @@ -409,13 +401,7 @@ abstract class AbstractServerTest { @Test fun `read module -- null contents and null error`() { - val reader = - ModuleReaderSpec( - scheme = "bird", - hasHierarchicalUris = true, - isLocal = true, - isGlobbable = false - ) + val reader = ModuleReaderSpec("bird", true, true, false) val evaluatorId = client.sendCreateEvaluatorRequest(moduleReaders = listOf(reader)) client.send( @@ -432,14 +418,7 @@ abstract class AbstractServerTest { assertThat(readModuleMsg.uri.toString()).isEqualTo("bird:/pigeon.pkl") assertThat(readModuleMsg.evaluatorId).isEqualTo(evaluatorId) - client.send( - ReadModuleResponse( - requestId = readModuleMsg.requestId, - evaluatorId = evaluatorId, - contents = null, - error = null - ) - ) + client.send(ReadModuleResponse(readModuleMsg.requestId, evaluatorId, null, null)) val evaluateResponse = client.receive() assertThat(evaluateResponse.error).isNull()