diff --git a/bench/gradle.lockfile b/bench/gradle.lockfile index 9cc4ebfc9..76ea17d8a 100644 --- a/bench/gradle.lockfile +++ b/bench/gradle.lockfile @@ -32,6 +32,7 @@ org.junit.jupiter:junit-jupiter-params:5.11.2=testCompileClasspath,testImplement org.junit.platform:junit-platform-commons:1.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata org.junit.platform:junit-platform-engine:1.11.2=testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata org.junit:junit-bom:5.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata +org.msgpack:msgpack-core:0.9.8=jmh,jmhRuntimeClasspath org.openjdk.jmh:jmh-core:1.37=jmh,jmhCompileClasspath,jmhImplementationDependenciesMetadata,jmhRuntimeClasspath org.openjdk.jmh:jmh-generator-asm:1.37=jmh,jmhCompileClasspath,jmhImplementationDependenciesMetadata,jmhRuntimeClasspath org.openjdk.jmh:jmh-generator-bytecode:1.37=jmh,jmhCompileClasspath,jmhImplementationDependenciesMetadata,jmhRuntimeClasspath diff --git a/docs/gradle.lockfile b/docs/gradle.lockfile index 3399a4d57..d3e9bdf98 100644 --- a/docs/gradle.lockfile +++ b/docs/gradle.lockfile @@ -30,6 +30,7 @@ org.junit.jupiter:junit-jupiter-params:5.11.2=testCompileClasspath,testImplement org.junit.platform:junit-platform-commons:1.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata org.junit.platform:junit-platform-engine:1.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata org.junit:junit-bom:5.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata +org.msgpack:msgpack-core:0.9.8=testRuntimeClasspath org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata org.organicdesign:Paguro:3.10.3=testRuntimeClasspath org.snakeyaml:snakeyaml-engine:2.5=testRuntimeClasspath diff --git a/docs/modules/bindings-specification/pages/message-passing-api.adoc b/docs/modules/bindings-specification/pages/message-passing-api.adoc index 638fd6b10..85e48d5dd 100644 --- a/docs/modules/bindings-specification/pages/message-passing-api.adoc +++ b/docs/modules/bindings-specification/pages/message-passing-api.adoc @@ -9,8 +9,8 @@ The first element of the array is a code that designates the message's type, enc The second element of the array is the message body, encoded as a map. Messages are passed between the _client_ and the _server_. -The _client_ is the host language (for example, the Swift application when using pkl-swift). -The _server_ is the entity that provides controls for interacting with Pkl. +When hosting Pkl (for example, the Swift application when using pkl-swift), the _client_ is the host program and the _server_ is the entity that provides controls for interacting with Pkl. +When implementing an xref:language-reference:index.adoc#external-readers[external reader], the _client_ is the external reader process and the _server_ is the Pkl evaluator. For example, in JSON representation: @@ -597,3 +597,96 @@ class PathElement { isDirectory: Boolean } ---- + +[[initialize-module-reader-request]] +=== Initialize Module Reader Request + +Code: `0x2e` + +Type: <> <> + +Initialize an xref:language-reference:index.adoc#external-readers[External Module Reader]. +This message is sent to external reader processes the first time a module scheme it is registered for is read. + +[source,pkl] +---- +/// A number identifying this request. +requestId: Int + +/// The module scheme to initialize. +scheme: String +---- + +[[initialize-module-reader-response]] +=== Initialize Module Reader Response + +Code: `0x2f` + +Type: <> <> + +Return the requested external module reader specification. +The `spec` field should be set to `null` when the external process does not implement the requested module scheme. + +[source,pkl] +---- +/// A number identifying this request. +requestId: Int + +/// Client-side module reader spec. +/// +/// Null when the external process does not implement the requested module scheme. +spec: ClientModuleReader? +---- + +`ClientModuleReader` is defined above by <>. + +[[initialize-resource-reader-request]] +=== Initialize Resource Reader Request + +Code: `0x30` + +Type: <> <> + +Initialize an xref:language-reference:index.adoc#external-readers[External Resource Reader]. +This message is sent to external reader processes the first time a resource scheme it is registered for is read. + +[source,pkl] +---- +/// A number identifying this request. +requestId: Int + +/// The resource scheme to initialize. +scheme: String +---- + +[[initialize-resource-reader-response]] +=== Initialize Resource Reader Response + +Code: `0x31` + +Type: <> <> + +Return the requested external resource reader specification. +The `spec` field should be set to `null` when the external process does not implement the requested resource scheme. + +[source,pkl] +---- +/// A number identifying this request. +requestId: Int + +/// Client-side resource reader spec. +/// +/// Null when the external process does not implement the requested resource scheme. +spec: ClientResourceReader? +---- + +`ClientResourceReader` is defined above by <>. + +[[close-external-process]] +=== Close External Process + +Code: `0x32` + +Type: <> <> + +Initiate graceful shutdown of the external reader process. + +[source,pkl] +---- +/// This message has no properties. +---- diff --git a/docs/modules/language-reference/pages/index.adoc b/docs/modules/language-reference/pages/index.adoc index bd0839aef..a40db97ff 100644 --- a/docs/modules/language-reference/pages/index.adoc +++ b/docs/modules/language-reference/pages/index.adoc @@ -3199,7 +3199,8 @@ This section discusses language features that are generally more relevant to tem <> + <> + <> + -<> +<> + +<> [[meaning-of-new]] === Meaning of `new` @@ -5465,3 +5466,68 @@ It can be imported using dependency notation, i.e. `import "@fruit/Pear.pkl"`. At runtime, it will resolve to relative path `../fruit/Pear.pkl`. When packaging projects with local dependencies, both the project and its dependent project must be passed to the xref:pkl-cli:index.adoc#command-project-package[`pkl project package`] command. + +[[external-readers]] +=== External Readers + +External readers are a mechanism to extend the <> and <> URI schemes that Pkl supports. +Readers are implemented as ordinary executables and use Pkl's xref:bindings-specification:message-passing-api.adoc[message passing API] to communicate with the hosting Pkl evaluator. +The xref:swift:ROOT:index.adoc[Swift] and xref:go:ROOT:index.adoc[Go] language binding libraries provide an `ExternalReaderRuntime` type to facilitate implementing external readers. + +External readers are configured separately for modules and resources. +They are registered by mapping their URI scheme to the executable to run and additonal arguments to pass. +This is done on the command line by passing `--external-resource-reader` and `--external-module-reader` flags, which may both be passed multiple times. + +[source,text] +---- +$ pkl eval --external-resource-reader = --external-module-reader =' ' +---- + +External readers may also be configured in a <> `PklProject` file. +[source,{pkl}] +---- +evaluatorSettings { + externalResourceReaders { + [""] { + executable = "" + } + } + externalModuleReaders { + [""] { + executable = "" + arguments { ""; "" } + } + } +} +---- + +Registering an external reader for a scheme automatically adds that scheme to the default allowed modules/resources. +As with Pkl's built-in module and resource schemes, setting explicit allowed modules or resources overrides this behavior and appropriate patterns must be specified to allow use of external readers. + +==== Example + +Consider this module: + +[source,{pkl}] +---- +username = "pigeon" + +email = read("ldap://ds.example.com:389/dc=example,dc=com?mail?sub?(uid=\(username))").text +---- + +Pkl doesn't implement the `ldap:` resource URI scheme natively, but an external reader can provide it. +Assuming a hypothetical `pkl-ldap` executable implementing the external reader protocol and the `ldap:` scheme is in the `$PATH`, this module can be evaluated as: + +[source,text] +---- +$ pkl eval --external-resource-reader ldap=pkl-ldap +username = "pigeon" +email = "pigeon@example.com" +---- + +In this example, the external reader may provide both `ldap:` and `ldaps:` schemes. +To support both schemes during evaluation, both would need to be registered explicitly: +[source,text] +---- +$ pkl eval --external-resource-reader ldap=pkl-ldap --external-resource-reader ldaps=pkl-ldap +---- diff --git a/docs/src/test/kotlin/DocSnippetTests.kt b/docs/src/test/kotlin/DocSnippetTests.kt index a0b1514d5..48873584f 100644 --- a/docs/src/test/kotlin/DocSnippetTests.kt +++ b/docs/src/test/kotlin/DocSnippetTests.kt @@ -22,10 +22,10 @@ import org.pkl.core.parser.antlr.PklParser import org.pkl.core.repl.ReplRequest import org.pkl.core.repl.ReplResponse import org.pkl.core.repl.ReplServer -import org.pkl.core.resource.ResourceReaders import org.pkl.core.util.IoUtils import org.antlr.v4.runtime.ParserRuleContext import org.pkl.core.http.HttpClient +import org.pkl.core.resource.ResourceReaders import java.nio.file.Files import kotlin.io.path.isDirectory import kotlin.io.path.isRegularFile diff --git a/pkl-cli/gradle.lockfile b/pkl-cli/gradle.lockfile index bc0011a3b..13ffb0a66 100644 --- a/pkl-cli/gradle.lockfile +++ b/pkl-cli/gradle.lockfile @@ -99,4 +99,4 @@ org.xmlunit:xmlunit-core:2.10.0=testCompileClasspath,testImplementationDependenc org.xmlunit:xmlunit-legacy:2.10.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath org.xmlunit:xmlunit-placeholders:2.10.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath org.yaml:snakeyaml:2.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath -empty=annotationProcessor,intransitiveDependenciesMetadata,javaExecutable,kotlinCompilerPluginClasspath,kotlinNativeCompilerPluginClasspath,kotlinScriptDef,kotlinScriptDefExtensions,runtimeOnlyDependenciesMetadata,shadow,signatures,sourcesJar,stagedAlpineLinuxAmd64Executable,stagedLinuxAarch64Executable,stagedLinuxAmd64Executable,stagedMacAarch64Executable,stagedMacAmd64Executable,stagedWindowsAmd64Executable,testAnnotationProcessor,testApiDependenciesMetadata,testCompileOnlyDependenciesMetadata,testIntransitiveDependenciesMetadata,testKotlinScriptDef,testKotlinScriptDefExtensions +empty=annotationProcessor,archives,compile,intransitiveDependenciesMetadata,javaExecutable,kotlinCompilerPluginClasspath,kotlinNativeCompilerPluginClasspath,kotlinScriptDef,kotlinScriptDefExtensions,runtime,runtimeOnlyDependenciesMetadata,shadow,signatures,sourcesJar,stagedAlpineLinuxAmd64Executable,stagedLinuxAarch64Executable,stagedLinuxAmd64Executable,stagedMacAarch64Executable,stagedMacAmd64Executable,stagedWindowsAmd64Executable,testAnnotationProcessor,testApiDependenciesMetadata,testCompile,testCompileOnly,testCompileOnlyDependenciesMetadata,testIntransitiveDependenciesMetadata,testKotlinScriptDef,testKotlinScriptDefExtensions,testRuntime diff --git a/pkl-cli/src/main/kotlin/org/pkl/cli/CliEvaluator.kt b/pkl-cli/src/main/kotlin/org/pkl/cli/CliEvaluator.kt index 6b71e6852..4301c327c 100644 --- a/pkl-cli/src/main/kotlin/org/pkl/cli/CliEvaluator.kt +++ b/pkl-cli/src/main/kotlin/org/pkl/cli/CliEvaluator.kt @@ -31,12 +31,12 @@ import org.pkl.commons.writeString import org.pkl.core.EvaluatorBuilder import org.pkl.core.ModuleSource import org.pkl.core.PklException -import org.pkl.core.module.ModuleKeyFactories import org.pkl.core.module.ModulePathResolver import org.pkl.core.runtime.ModuleResolver import org.pkl.core.runtime.VmException import org.pkl.core.runtime.VmUtils import org.pkl.core.util.IoUtils +import org.pkl.core.util.Readers private data class OutputFile(val pathSpec: String, val moduleUri: URI) @@ -100,7 +100,8 @@ constructor( writeOutput(builder) } } finally { - ModuleKeyFactories.closeQuietly(builder.moduleKeyFactories) + Readers.closeQuietly(builder.moduleKeyFactories) + Readers.closeQuietly(builder.resourceReaders) } } diff --git a/pkl-cli/src/main/kotlin/org/pkl/cli/CliImportAnalyzer.kt b/pkl-cli/src/main/kotlin/org/pkl/cli/CliImportAnalyzer.kt index 7a94ab495..1463216ae 100644 --- a/pkl-cli/src/main/kotlin/org/pkl/cli/CliImportAnalyzer.kt +++ b/pkl-cli/src/main/kotlin/org/pkl/cli/CliImportAnalyzer.kt @@ -20,7 +20,7 @@ import org.pkl.commons.cli.CliCommand import org.pkl.commons.createParentDirectories import org.pkl.commons.writeString import org.pkl.core.ModuleSource -import org.pkl.core.module.ModuleKeyFactories +import org.pkl.core.util.Readers class CliImportAnalyzer @JvmOverloads @@ -73,7 +73,8 @@ constructor( .build() .use { it.evaluateOutputText(sourceModule) } } finally { - ModuleKeyFactories.closeQuietly(builder.moduleKeyFactories) + Readers.closeQuietly(builder.moduleKeyFactories) + Readers.closeQuietly(builder.resourceReaders) } } } diff --git a/pkl-cli/src/main/kotlin/org/pkl/cli/CliServer.kt b/pkl-cli/src/main/kotlin/org/pkl/cli/CliServer.kt index c501011a5..1a83039a7 100644 --- a/pkl-cli/src/main/kotlin/org/pkl/cli/CliServer.kt +++ b/pkl-cli/src/main/kotlin/org/pkl/cli/CliServer.kt @@ -18,14 +18,13 @@ package org.pkl.cli import org.pkl.commons.cli.CliBaseOptions import org.pkl.commons.cli.CliCommand import org.pkl.commons.cli.CliException -import org.pkl.server.MessageTransports -import org.pkl.server.ProtocolException +import org.pkl.core.messaging.ProtocolException import org.pkl.server.Server class CliServer(options: CliBaseOptions) : CliCommand(options) { override fun doRun() = try { - val server = Server(MessageTransports.stream(System.`in`, System.out)) + val server = Server.stream(System.`in`, System.out) server.use { it.start() } } catch (e: ProtocolException) { throw CliException(e.message!!) diff --git a/pkl-cli/src/main/kotlin/org/pkl/cli/CliTestRunner.kt b/pkl-cli/src/main/kotlin/org/pkl/cli/CliTestRunner.kt index 4ce530e0f..24a87761a 100644 --- a/pkl-cli/src/main/kotlin/org/pkl/cli/CliTestRunner.kt +++ b/pkl-cli/src/main/kotlin/org/pkl/cli/CliTestRunner.kt @@ -19,10 +19,10 @@ import java.io.Writer import org.pkl.commons.cli.* import org.pkl.core.EvaluatorBuilder import org.pkl.core.ModuleSource.uri -import org.pkl.core.module.ModuleKeyFactories import org.pkl.core.stdlib.test.report.JUnitReport import org.pkl.core.stdlib.test.report.SimpleReport import org.pkl.core.util.ErrorMessages +import org.pkl.core.util.Readers class CliTestRunner @JvmOverloads @@ -38,7 +38,8 @@ constructor( try { evalTest(builder) } finally { - ModuleKeyFactories.closeQuietly(builder.moduleKeyFactories) + Readers.closeQuietly(builder.moduleKeyFactories) + Readers.closeQuietly(builder.resourceReaders) } } diff --git a/pkl-codegen-java/gradle.lockfile b/pkl-codegen-java/gradle.lockfile index e937253e7..13711d702 100644 --- a/pkl-codegen-java/gradle.lockfile +++ b/pkl-codegen-java/gradle.lockfile @@ -33,6 +33,7 @@ org.junit.jupiter:junit-jupiter-params:5.11.2=testCompileClasspath,testImplement org.junit.platform:junit-platform-commons:1.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata org.junit.platform:junit-platform-engine:1.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata org.junit:junit-bom:5.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata +org.msgpack:msgpack-core:0.9.8=runtimeClasspath,testRuntimeClasspath org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata org.organicdesign:Paguro:3.10.3=runtimeClasspath,testRuntimeClasspath org.snakeyaml:snakeyaml-engine:2.5=runtimeClasspath,testRuntimeClasspath diff --git a/pkl-codegen-java/src/main/kotlin/org/pkl/codegen/java/CliJavaCodeGenerator.kt b/pkl-codegen-java/src/main/kotlin/org/pkl/codegen/java/CliJavaCodeGenerator.kt index f8a88c5dd..5498448bb 100644 --- a/pkl-codegen-java/src/main/kotlin/org/pkl/codegen/java/CliJavaCodeGenerator.kt +++ b/pkl-codegen-java/src/main/kotlin/org/pkl/codegen/java/CliJavaCodeGenerator.kt @@ -21,7 +21,7 @@ import org.pkl.commons.cli.CliException import org.pkl.commons.createParentDirectories import org.pkl.commons.writeString import org.pkl.core.ModuleSource -import org.pkl.core.module.ModuleKeyFactories +import org.pkl.core.util.Readers /** API for the Java code generator CLI. */ class CliJavaCodeGenerator(private val options: CliJavaCodeGeneratorOptions) : @@ -49,7 +49,8 @@ class CliJavaCodeGenerator(private val options: CliJavaCodeGeneratorOptions) : } } } finally { - ModuleKeyFactories.closeQuietly(builder.moduleKeyFactories) + Readers.closeQuietly(builder.moduleKeyFactories) + Readers.closeQuietly(builder.resourceReaders) } } } diff --git a/pkl-codegen-kotlin/gradle.lockfile b/pkl-codegen-kotlin/gradle.lockfile index 87726a9f5..94efff416 100644 --- a/pkl-codegen-kotlin/gradle.lockfile +++ b/pkl-codegen-kotlin/gradle.lockfile @@ -36,6 +36,7 @@ org.junit.jupiter:junit-jupiter-params:5.11.2=testCompileClasspath,testImplement org.junit.platform:junit-platform-commons:1.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata org.junit.platform:junit-platform-engine:1.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata org.junit:junit-bom:5.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata +org.msgpack:msgpack-core:0.9.8=runtimeClasspath,testRuntimeClasspath org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata org.organicdesign:Paguro:3.10.3=runtimeClasspath,testRuntimeClasspath org.snakeyaml:snakeyaml-engine:2.5=runtimeClasspath,testRuntimeClasspath diff --git a/pkl-codegen-kotlin/src/main/kotlin/org/pkl/codegen/kotlin/CliKotlinCodeGenerator.kt b/pkl-codegen-kotlin/src/main/kotlin/org/pkl/codegen/kotlin/CliKotlinCodeGenerator.kt index e6da1bfb5..4c0681807 100644 --- a/pkl-codegen-kotlin/src/main/kotlin/org/pkl/codegen/kotlin/CliKotlinCodeGenerator.kt +++ b/pkl-codegen-kotlin/src/main/kotlin/org/pkl/codegen/kotlin/CliKotlinCodeGenerator.kt @@ -21,7 +21,7 @@ import org.pkl.commons.cli.CliException import org.pkl.commons.createParentDirectories import org.pkl.commons.writeString import org.pkl.core.ModuleSource -import org.pkl.core.module.ModuleKeyFactories +import org.pkl.core.util.Readers /** API for the Kotlin code generator CLI. */ class CliKotlinCodeGenerator(private val options: CliKotlinCodeGeneratorOptions) : @@ -50,7 +50,8 @@ class CliKotlinCodeGenerator(private val options: CliKotlinCodeGeneratorOptions) } } } finally { - ModuleKeyFactories.closeQuietly(builder.moduleKeyFactories) + Readers.closeQuietly(builder.moduleKeyFactories) + Readers.closeQuietly(builder.resourceReaders) } } } diff --git a/pkl-commons-cli/gradle.lockfile b/pkl-commons-cli/gradle.lockfile index 7ca482dc1..e3d61b289 100644 --- a/pkl-commons-cli/gradle.lockfile +++ b/pkl-commons-cli/gradle.lockfile @@ -31,6 +31,7 @@ org.junit.jupiter:junit-jupiter-params:5.11.2=testCompileClasspath,testImplement org.junit.platform:junit-platform-commons:1.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata org.junit.platform:junit-platform-engine:1.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata org.junit:junit-bom:5.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata +org.msgpack:msgpack-core:0.9.8=runtimeClasspath,testRuntimeClasspath org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata org.organicdesign:Paguro:3.10.3=runtimeClasspath,testRuntimeClasspath org.snakeyaml:snakeyaml-engine:2.5=runtimeClasspath,testRuntimeClasspath diff --git a/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliBaseOptions.kt b/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliBaseOptions.kt index 7eb9b1eb4..e7f52b03b 100644 --- a/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliBaseOptions.kt +++ b/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliBaseOptions.kt @@ -20,6 +20,7 @@ import java.nio.file.Files import java.nio.file.Path import java.time.Duration import java.util.regex.Pattern +import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader import org.pkl.core.module.ProjectDependenciesManager import org.pkl.core.util.IoUtils @@ -134,6 +135,12 @@ data class CliBaseOptions( /** Hostnames, IP addresses, or CIDR blocks to not proxy. */ val httpNoProxy: List? = null, + + /** External module reader process specs */ + val externalModuleReaders: Map = mapOf(), + + /** External resource reader process specs */ + val externalResourceReaders: Map = mapOf(), ) { companion object { diff --git a/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliCommand.kt b/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliCommand.kt index 5020ee031..15759fbf3 100644 --- a/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliCommand.kt +++ b/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliCommand.kt @@ -21,6 +21,7 @@ import java.util.regex.Pattern import kotlin.io.path.isRegularFile import org.pkl.core.* import org.pkl.core.evaluatorSettings.PklEvaluatorSettings +import org.pkl.core.externalreader.ExternalReaderProcessImpl import org.pkl.core.http.HttpClient import org.pkl.core.module.ModuleKeyFactories import org.pkl.core.module.ModuleKeyFactory @@ -108,12 +109,16 @@ abstract class CliCommand(protected val cliOptions: CliBaseOptions) { protected val allowedModules: List by lazy { cliOptions.allowedModules - ?: evaluatorSettings?.allowedModules ?: SecurityManagers.defaultAllowedModules + ?: evaluatorSettings?.allowedModules + ?: (SecurityManagers.defaultAllowedModules + + externalModuleReaders.keys.map { Pattern.compile("$it:") }.toList()) } protected val allowedResources: List by lazy { cliOptions.allowedResources - ?: evaluatorSettings?.allowedResources ?: SecurityManagers.defaultAllowedResources + ?: evaluatorSettings?.allowedResources + ?: (SecurityManagers.defaultAllowedResources + + externalResourceReaders.keys.map { Pattern.compile("$it:") }.toList()) } protected val rootDir: Path? by lazy { @@ -169,6 +174,26 @@ abstract class CliCommand(protected val cliOptions: CliBaseOptions) { ?: project?.evaluatorSettings?.http?.proxy?.noProxy ?: settings.http?.proxy?.noProxy } + private val externalModuleReaders by lazy { + (project?.evaluatorSettings?.externalModuleReaders + ?: emptyMap()) + cliOptions.externalModuleReaders + } + + private val externalResourceReaders by lazy { + (project?.evaluatorSettings?.externalResourceReaders + ?: emptyMap()) + cliOptions.externalResourceReaders + } + + private val externalProcesses by lazy { + // share ExternalProcessImpl instances between configured external resource/module readers with + // the same spec + // this avoids spawning multiple subprocesses if the same reader implements both reader types + // and/or multiple schemes + (externalModuleReaders + externalResourceReaders).values.toSet().associateWith { + ExternalReaderProcessImpl(it) + } + } + private fun HttpClient.Builder.addDefaultCliCertificates() { val caCertsDir = IoUtils.getPklHomeDir().resolve("cacerts") var certsAdded = false @@ -213,6 +238,9 @@ abstract class CliCommand(protected val cliOptions: CliBaseOptions) { protected fun moduleKeyFactories(modulePathResolver: ModulePathResolver): List { return buildList { + externalModuleReaders.forEach { (key, value) -> + add(ModuleKeyFactories.externalProcess(key, externalProcesses[value]!!)) + } add(ModuleKeyFactories.standardLibrary) add(ModuleKeyFactories.modulePath(modulePathResolver)) add(ModuleKeyFactories.pkg) @@ -234,6 +262,9 @@ abstract class CliCommand(protected val cliOptions: CliBaseOptions) { add(ResourceReaders.file()) add(ResourceReaders.http()) add(ResourceReaders.https()) + externalResourceReaders.forEach { (key, value) -> + add(ResourceReaders.externalProcess(key, externalProcesses[value]!!)) + } } } diff --git a/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/commands/BaseOptions.kt b/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/commands/BaseOptions.kt index 1f84451f2..77c2ed7cb 100644 --- a/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/commands/BaseOptions.kt +++ b/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/commands/BaseOptions.kt @@ -28,6 +28,8 @@ import java.time.Duration import java.util.regex.Pattern import org.pkl.commons.cli.CliBaseOptions import org.pkl.commons.cli.CliException +import org.pkl.commons.shlex +import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader import org.pkl.core.runtime.VmUtils import org.pkl.core.util.IoUtils @@ -74,6 +76,17 @@ class BaseOptions : OptionGroup() { .multiple() .toMap() } + + fun OptionWithValues.parseExternalReader( + delimiter: String + ): OptionWithValues< + Pair?, Pair, Pair + > { + return splitPair(delimiter).convert { + val cmd = shlex(it.second) + Pair(it.first, ExternalReader(cmd.first(), cmd.drop(1))) + } + } } private val defaults = CliBaseOptions() @@ -207,6 +220,26 @@ class BaseOptions : OptionGroup() { .single() .split(",") + val externalModuleReaders: Map by + option( + names = arrayOf("--external-module-reader"), + metavar = "='[ ]'", + help = "External reader registrations for module URI schemes" + ) + .parseExternalReader("=") + .multiple() + .toMap() + + val externalResourceReaders: Map by + option( + names = arrayOf("--external-resource-reader"), + metavar = "='[ ]'", + help = "External reader registrations for resource URI schemes" + ) + .parseExternalReader("=") + .multiple() + .toMap() + // hidden option used by native tests private val testPort: Int by option(names = arrayOf("--test-port"), help = "Internal test option", hidden = true) @@ -239,7 +272,9 @@ class BaseOptions : OptionGroup() { noProject = projectOptions?.noProject ?: false, caCertificates = caCertificates, httpProxy = proxy, - httpNoProxy = noProxy ?: emptyList() + httpNoProxy = noProxy ?: emptyList(), + externalModuleReaders = externalModuleReaders, + externalResourceReaders = externalResourceReaders, ) } } diff --git a/pkl-commons-cli/src/test/kotlin/org/pkl/commons/cli/BaseCommandTest.kt b/pkl-commons-cli/src/test/kotlin/org/pkl/commons/cli/BaseCommandTest.kt index a464fc2e3..759487c69 100644 --- a/pkl-commons-cli/src/test/kotlin/org/pkl/commons/cli/BaseCommandTest.kt +++ b/pkl-commons-cli/src/test/kotlin/org/pkl/commons/cli/BaseCommandTest.kt @@ -24,6 +24,7 @@ import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows import org.pkl.commons.cli.commands.BaseCommand +import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader class BaseCommandTest { @@ -72,4 +73,34 @@ class BaseCommandTest { assertThat(cmd.baseOptions.allowedResources).isEmpty() } + + @Test + fun `--external-resource-reader and --external-module-reader are parsed correctly`() { + cmd.parse( + arrayOf( + "--external-module-reader", + "scheme3=reader3", + "--external-module-reader", + "scheme4=reader4 with args", + "--external-resource-reader", + "scheme1=reader1", + "--external-resource-reader", + "scheme2=reader2 with args" + ) + ) + assertThat(cmd.baseOptions.externalModuleReaders) + .isEqualTo( + mapOf( + "scheme3" to ExternalReader("reader3", emptyList()), + "scheme4" to ExternalReader("reader4", listOf("with", "args")) + ) + ) + assertThat(cmd.baseOptions.externalResourceReaders) + .isEqualTo( + mapOf( + "scheme1" to ExternalReader("reader1", emptyList()), + "scheme2" to ExternalReader("reader2", listOf("with", "args")) + ) + ) + } } diff --git a/pkl-commons/src/main/kotlin/org/pkl/commons/Strings.kt b/pkl-commons/src/main/kotlin/org/pkl/commons/Strings.kt index c37d7efe9..d4d8e1728 100644 --- a/pkl-commons/src/main/kotlin/org/pkl/commons/Strings.kt +++ b/pkl-commons/src/main/kotlin/org/pkl/commons/Strings.kt @@ -18,6 +18,7 @@ package org.pkl.commons import java.io.File import java.net.URI import java.nio.file.Path +import java.util.* import java.util.regex.Pattern fun String.toPath(): Path = Path.of(this) @@ -36,3 +37,53 @@ fun String.toUri(): URI { } return URI(null, null, this, null) } + +/** Lex a string into tokens similar to how a shell would */ +fun shlex(input: String): List { + val result = mutableListOf() + var inEscape = false + var quote: Char? = null + var lastCloseQuoteIndex = Int.MIN_VALUE + val current = StringBuilder() + + for ((idx, char) in input.withIndex()) { + when { + // if in an escape always append the next character + inEscape -> { + inEscape = false + current.append(char) + } + // enter an escape on \ if not in a quote or in a non-single quote + char == '\\' && quote != '\'' -> inEscape = true + // if in a quote and encounter the delimiter, tentatively exit the quote + // this handles cases with adjoining quotes e.g. `abc'123''xyz'` + quote == char -> { + quote = null + lastCloseQuoteIndex = idx + } + // if not in a quote and encounter a quote charater, enter a quote + quote == null && (char == '\'' || char == '"') -> { + quote = char + } + // if not in a quote and whitespace is encountered + quote == null && char.isWhitespace() -> { + // if the current token isn't empty or if a quote has just ended, finalize the current token + // otherwise do nothing, which handles multiple whitespace cases e.g. `abc 123` + if (current.isNotEmpty() || lastCloseQuoteIndex == (idx - 1)) { + result.add(current.toString()) + current.clear() + } + } + // in other cases, append to the current token + else -> current.append(char) + } + } + // clean up last token + // if the current token isn't empty or if a quote has just ended, finalize the token + // if this condition is false, the input likely ended in whitespace + if (current.isNotEmpty() || lastCloseQuoteIndex == (input.length - 1)) { + result.add(current.toString()) + } + + return result +} diff --git a/pkl-commons/src/test/kotlin/org/pkl/commons/ShlexTest.kt b/pkl-commons/src/test/kotlin/org/pkl/commons/ShlexTest.kt new file mode 100644 index 000000000..bd76c388a --- /dev/null +++ b/pkl-commons/src/test/kotlin/org/pkl/commons/ShlexTest.kt @@ -0,0 +1,78 @@ +/* + * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.commons + +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test + +class ShlexTest { + + @Test + fun `empty input produces empty output`() { + assertThat(shlex("")).isEqualTo(emptyList()) + } + + @Test + fun `whitespace input produces empty output`() { + assertThat(shlex(" \n \t ")).isEqualTo(emptyList()) + } + + @Test + fun `regular token parsing`() { + assertThat(shlex("\nabc def\tghi ")).isEqualTo(listOf("abc", "def", "ghi")) + } + + @Test + fun `single quoted token parsing`() { + assertThat(shlex("'this is a single token'")).isEqualTo(listOf("this is a single token")) + } + + @Test + fun `double quoted token parsing`() { + assertThat(shlex("\"this is a single token\"")).isEqualTo(listOf("this is a single token")) + } + + @Test + fun `escaping handles double quotes`() { + assertThat(shlex(""""\"this is a single double quoted token\""""")) + .isEqualTo(listOf("\"this is a single double quoted token\"")) + } + + @Test + fun `escaping does not apply within single quotes`() { + assertThat(shlex("""'this is a single \" token'""")) + .isEqualTo(listOf("""this is a single \" token""")) + } + + @Test + fun `adjacent quoted strings are one token`() { + assertThat(shlex(""""single"' joined 'token""")).isEqualTo(listOf("single joined token")) + assertThat(shlex(""""single"' 'token""")).isEqualTo(listOf("single token")) + } + + @Test + fun `space escapes do not split tokens`() { + assertThat(shlex("""single\ token""")).isEqualTo(listOf("single token")) + } + + @Test + fun `empty quotes produce a single empty token`() { + assertThat(shlex("\"\"")).isEqualTo(listOf("")) + assertThat(shlex("''")).isEqualTo(listOf("")) + assertThat(shlex("'' ''")).isEqualTo(listOf("", "")) + assertThat(shlex("''''")).isEqualTo(listOf("")) + } +} diff --git a/pkl-config-java/gradle.lockfile b/pkl-config-java/gradle.lockfile index 30126d728..79cf66214 100644 --- a/pkl-config-java/gradle.lockfile +++ b/pkl-config-java/gradle.lockfile @@ -34,6 +34,7 @@ org.junit.jupiter:junit-jupiter-params:5.11.2=testCompileClasspath,testImplement org.junit.platform:junit-platform-commons:1.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata org.junit.platform:junit-platform-engine:1.11.2=testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata org.junit:junit-bom:5.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata +org.msgpack:msgpack-core:0.9.8=pklCodegenJava,runtimeClasspath,testRuntimeClasspath org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata org.organicdesign:Paguro:3.10.3=pklCodegenJava,runtimeClasspath,testRuntimeClasspath org.snakeyaml:snakeyaml-engine:2.5=pklCodegenJava,runtimeClasspath,testRuntimeClasspath diff --git a/pkl-config-kotlin/gradle.lockfile b/pkl-config-kotlin/gradle.lockfile index 518f09bad..b3ae14a53 100644 --- a/pkl-config-kotlin/gradle.lockfile +++ b/pkl-config-kotlin/gradle.lockfile @@ -33,6 +33,7 @@ org.junit.jupiter:junit-jupiter-params:5.11.2=testCompileClasspath,testImplement org.junit.platform:junit-platform-commons:1.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata org.junit.platform:junit-platform-engine:1.11.2=testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata org.junit:junit-bom:5.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata +org.msgpack:msgpack-core:0.9.8=pklCodegenKotlin,pklConfigJava,runtimeClasspath,testRuntimeClasspath org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata org.organicdesign:Paguro:3.10.3=pklCodegenKotlin,pklConfigJava,runtimeClasspath,testRuntimeClasspath org.snakeyaml:snakeyaml-engine:2.5=pklCodegenKotlin,pklConfigJava,runtimeClasspath,testRuntimeClasspath diff --git a/pkl-core/gradle.lockfile b/pkl-core/gradle.lockfile index 8dd0f5225..3d06ea3df 100644 --- a/pkl-core/gradle.lockfile +++ b/pkl-core/gradle.lockfile @@ -37,6 +37,7 @@ org.junit.jupiter:junit-jupiter-params:5.11.2=testCompileClasspath,testImplement org.junit.platform:junit-platform-commons:1.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata org.junit.platform:junit-platform-engine:1.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata org.junit:junit-bom:5.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata +org.msgpack:msgpack-core:0.9.8=compileClasspath,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata org.organicdesign:Paguro:3.10.3=compileClasspath,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath org.snakeyaml:snakeyaml-engine:2.5=compileClasspath,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath diff --git a/pkl-core/pkl-core.gradle.kts b/pkl-core/pkl-core.gradle.kts index 275a396b5..869690209 100644 --- a/pkl-core/pkl-core.gradle.kts +++ b/pkl-core/pkl-core.gradle.kts @@ -57,6 +57,7 @@ dependencies { compileOnly(projects.pklExecutor) implementation(libs.antlrRuntime) + implementation(libs.msgpack) implementation(libs.truffleApi) implementation(libs.graalSdk) diff --git a/pkl-core/src/main/java/org/pkl/core/EvaluatorBuilder.java b/pkl-core/src/main/java/org/pkl/core/EvaluatorBuilder.java index a76224439..8d797bc2d 100644 --- a/pkl-core/src/main/java/org/pkl/core/EvaluatorBuilder.java +++ b/pkl-core/src/main/java/org/pkl/core/EvaluatorBuilder.java @@ -20,6 +20,9 @@ import java.util.*; import java.util.regex.Pattern; import org.pkl.core.SecurityManagers.StandardBuilder; +import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader; +import org.pkl.core.externalreader.ExternalReaderProcess; +import org.pkl.core.externalreader.ExternalReaderProcessImpl; import org.pkl.core.http.HttpClient; import org.pkl.core.module.ModuleKeyFactories; import org.pkl.core.module.ModuleKeyFactory; @@ -478,6 +481,25 @@ public EvaluatorBuilder applyFromProject(Project project) { } else if (settings.moduleCacheDir() != null) { setModuleCacheDir(settings.moduleCacheDir()); } + + // this isn't ideal as project and non-project ExternalProcessImpl instances can be dupes + var procs = new HashMap(); + if (settings.externalModuleReaders() != null) { + for (var entry : settings.externalModuleReaders().entrySet()) { + addModuleKeyFactory( + ModuleKeyFactories.externalProcess( + entry.getKey(), + procs.computeIfAbsent(entry.getValue(), ExternalReaderProcessImpl::new))); + } + } + if (settings.externalResourceReaders() != null) { + for (var entry : settings.externalResourceReaders().entrySet()) { + addResourceReader( + ResourceReaders.externalProcess( + entry.getKey(), + procs.computeIfAbsent(entry.getValue(), ExternalReaderProcessImpl::new))); + } + } return this; } diff --git a/pkl-core/src/main/java/org/pkl/core/ast/builder/AstBuilder.java b/pkl-core/src/main/java/org/pkl/core/ast/builder/AstBuilder.java index e4e86926f..44b125804 100644 --- a/pkl-core/src/main/java/org/pkl/core/ast/builder/AstBuilder.java +++ b/pkl-core/src/main/java/org/pkl/core/ast/builder/AstBuilder.java @@ -55,6 +55,7 @@ import org.pkl.core.ast.lambda.ApplyVmFunction1NodeGen; import org.pkl.core.ast.member.*; import org.pkl.core.ast.type.*; +import org.pkl.core.externalreader.ExternalReaderProcessException; import org.pkl.core.module.ModuleKey; import org.pkl.core.module.ModuleKeys; import org.pkl.core.module.ResolvedModuleKey; @@ -1847,6 +1848,12 @@ private URI resolveImport(String importUri, StringConstantContext importUriCtx) .withHint(e.getHint()) .withSourceSection(createSourceSection(importUriCtx)) .build(); + } catch (ExternalReaderProcessException e) { + throw exceptionBuilder() + .evalError("externalReaderFailure") + .withCause(e.getCause()) + .withSourceSection(createSourceSection(importUriCtx)) + .build(); } if (!resolvedUri.isAbsolute()) { diff --git a/pkl-core/src/main/java/org/pkl/core/ast/expression/unary/AbstractReadNode.java b/pkl-core/src/main/java/org/pkl/core/ast/expression/unary/AbstractReadNode.java index 45596793f..607e04189 100644 --- a/pkl-core/src/main/java/org/pkl/core/ast/expression/unary/AbstractReadNode.java +++ b/pkl-core/src/main/java/org/pkl/core/ast/expression/unary/AbstractReadNode.java @@ -23,6 +23,7 @@ import java.net.URI; import java.net.URISyntaxException; import org.pkl.core.SecurityManagerException; +import org.pkl.core.externalreader.ExternalReaderProcessException; import org.pkl.core.module.ModuleKey; import org.pkl.core.packages.PackageLoadError; import org.pkl.core.runtime.VmContext; @@ -75,6 +76,8 @@ private URI resolveResource(ModuleKey moduleKey, String resourceUri) { .build(); } catch (PackageLoadError | SecurityManagerException e) { throw exceptionBuilder().withCause(e).build(); + } catch (ExternalReaderProcessException e) { + throw exceptionBuilder().evalError("externalReaderFailure").withCause(e).build(); } if (!resolvedUri.isAbsolute()) { diff --git a/pkl-core/src/main/java/org/pkl/core/ast/expression/unary/ImportGlobNode.java b/pkl-core/src/main/java/org/pkl/core/ast/expression/unary/ImportGlobNode.java index 2404c754b..66ae35934 100644 --- a/pkl-core/src/main/java/org/pkl/core/ast/expression/unary/ImportGlobNode.java +++ b/pkl-core/src/main/java/org/pkl/core/ast/expression/unary/ImportGlobNode.java @@ -25,6 +25,7 @@ import java.net.URI; import org.pkl.core.SecurityManagerException; import org.pkl.core.ast.member.SharedMemberNode; +import org.pkl.core.externalreader.ExternalReaderProcessException; import org.pkl.core.http.HttpClientInitException; import org.pkl.core.module.ResolvedModuleKey; import org.pkl.core.packages.PackageLoadError; @@ -104,6 +105,8 @@ public Object executeGeneric(VirtualFrame frame) { .evalError("invalidGlobPattern", globPattern) .withHint(e.getMessage()) .build(); + } catch (ExternalReaderProcessException e) { + throw exceptionBuilder().evalError("externalReaderFailure").withCause(e).build(); } } } diff --git a/pkl-core/src/main/java/org/pkl/core/ast/expression/unary/ReadGlobNode.java b/pkl-core/src/main/java/org/pkl/core/ast/expression/unary/ReadGlobNode.java index 64e369722..9331629bc 100644 --- a/pkl-core/src/main/java/org/pkl/core/ast/expression/unary/ReadGlobNode.java +++ b/pkl-core/src/main/java/org/pkl/core/ast/expression/unary/ReadGlobNode.java @@ -26,6 +26,7 @@ import org.graalvm.collections.EconomicMap; import org.pkl.core.SecurityManagerException; import org.pkl.core.ast.member.SharedMemberNode; +import org.pkl.core.externalreader.ExternalReaderProcessException; import org.pkl.core.http.HttpClientInitException; import org.pkl.core.module.ModuleKey; import org.pkl.core.runtime.VmContext; @@ -103,6 +104,8 @@ public Object read(String globPattern) { .evalError("invalidGlobPattern", globPattern) .withHint(e.getMessage()) .build(); + } catch (ExternalReaderProcessException e) { + throw exceptionBuilder().evalError("externalReaderFailure").withCause(e).build(); } } } diff --git a/pkl-core/src/main/java/org/pkl/core/evaluatorSettings/PklEvaluatorSettings.java b/pkl-core/src/main/java/org/pkl/core/evaluatorSettings/PklEvaluatorSettings.java index 385e53418..946c5a993 100644 --- a/pkl-core/src/main/java/org/pkl/core/evaluatorSettings/PklEvaluatorSettings.java +++ b/pkl-core/src/main/java/org/pkl/core/evaluatorSettings/PklEvaluatorSettings.java @@ -20,9 +20,11 @@ import java.nio.file.Path; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Objects; import java.util.function.BiFunction; import java.util.regex.Pattern; +import java.util.stream.Collectors; import org.pkl.core.Duration; import org.pkl.core.PNull; import org.pkl.core.PObject; @@ -43,7 +45,9 @@ public record PklEvaluatorSettings( @Nullable List modulePath, @Nullable Duration timeout, @Nullable Path rootDir, - @Nullable Http http) { + @Nullable Http http, + @Nullable Map externalModuleReaders, + @Nullable Map externalResourceReaders) { /** Initializes a {@link PklEvaluatorSettings} from a raw object representation. */ @SuppressWarnings("unchecked") @@ -80,6 +84,24 @@ public static PklEvaluatorSettings parse( var rootDirStr = (String) pSettings.get("rootDir"); var rootDir = rootDirStr == null ? null : pathNormalizer.apply(rootDirStr, "rootDir"); + var externalModuleReadersRaw = (Map) pSettings.get("externalModuleReaders"); + var externalModuleReaders = + externalModuleReadersRaw == null + ? null + : externalModuleReadersRaw.entrySet().stream() + .collect( + Collectors.toMap( + Entry::getKey, entry -> ExternalReader.parse(entry.getValue()))); + + var externalResourceReadersRaw = (Map) pSettings.get("externalResourceReaders"); + var externalResourceReaders = + externalResourceReadersRaw == null + ? null + : externalResourceReadersRaw.entrySet().stream() + .collect( + Collectors.toMap( + Entry::getKey, entry -> ExternalReader.parse(entry.getValue()))); + return new PklEvaluatorSettings( (Map) pSettings.get("externalProperties"), (Map) pSettings.get("env"), @@ -90,7 +112,9 @@ public static PklEvaluatorSettings parse( modulePath, (Duration) pSettings.get("timeout"), rootDir, - Http.parse((Value) pSettings.get("http"))); + Http.parse((Value) pSettings.get("http")), + externalModuleReaders, + externalResourceReaders); } public record Http(@Nullable Proxy proxy) { @@ -133,6 +157,18 @@ public static Proxy create(@Nullable String address, @Nullable List noPr } } + public record ExternalReader(String executable, @Nullable List arguments) { + @SuppressWarnings("unchecked") + public static ExternalReader parse(Value input) { + if (input instanceof PObject externalReader) { + var executable = (String) externalReader.getProperty("executable"); + var arguments = (List) externalReader.get("arguments"); + return new ExternalReader(executable, arguments); + } + throw PklBugException.unreachableCode(); + } + } + private boolean arePatternsEqual( @Nullable List thesePatterns, @Nullable List thosePatterns) { if (thesePatterns == null) { diff --git a/pkl-core/src/main/java/org/pkl/core/externalreader/ExternalReaderMessagePackDecoder.java b/pkl-core/src/main/java/org/pkl/core/externalreader/ExternalReaderMessagePackDecoder.java new file mode 100644 index 000000000..5855e2e5c --- /dev/null +++ b/pkl-core/src/main/java/org/pkl/core/externalreader/ExternalReaderMessagePackDecoder.java @@ -0,0 +1,61 @@ +/* + * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.core.externalreader; + +import java.io.InputStream; +import java.net.URISyntaxException; +import java.util.Map; +import org.msgpack.core.MessagePack; +import org.msgpack.core.MessageUnpacker; +import org.msgpack.value.Value; +import org.pkl.core.externalreader.ExternalReaderMessages.*; +import org.pkl.core.messaging.BaseMessagePackDecoder; +import org.pkl.core.messaging.DecodeException; +import org.pkl.core.messaging.Message; +import org.pkl.core.messaging.Message.Type; +import org.pkl.core.util.Nullable; + +public class ExternalReaderMessagePackDecoder extends BaseMessagePackDecoder { + + public ExternalReaderMessagePackDecoder(MessageUnpacker unpacker) { + super(unpacker); + } + + public ExternalReaderMessagePackDecoder(InputStream inputStream) { + this(MessagePack.newDefaultUnpacker(inputStream)); + } + + @Override + protected @Nullable Message decodeMessage(Type msgType, Map map) + throws DecodeException, URISyntaxException { + return switch (msgType) { + case INITIALIZE_MODULE_READER_REQUEST -> + new InitializeModuleReaderRequest( + unpackLong(map, "requestId"), unpackString(map, "scheme")); + case INITIALIZE_RESOURCE_READER_REQUEST -> + new InitializeResourceReaderRequest( + unpackLong(map, "requestId"), unpackString(map, "scheme")); + case INITIALIZE_MODULE_READER_RESPONSE -> + new InitializeModuleReaderResponse( + unpackLong(map, "requestId"), unpackModuleReaderSpec(getNullable(map, "spec"))); + case INITIALIZE_RESOURCE_READER_RESPONSE -> + new InitializeResourceReaderResponse( + unpackLong(map, "requestId"), unpackResourceReaderSpec(getNullable(map, "spec"))); + case CLOSE_EXTERNAL_PROCESS -> new CloseExternalProcess(); + default -> super.decodeMessage(msgType, map); + }; + } +} diff --git a/pkl-core/src/main/java/org/pkl/core/externalreader/ExternalReaderMessagePackEncoder.java b/pkl-core/src/main/java/org/pkl/core/externalreader/ExternalReaderMessagePackEncoder.java new file mode 100644 index 000000000..2a8ae3fa9 --- /dev/null +++ b/pkl-core/src/main/java/org/pkl/core/externalreader/ExternalReaderMessagePackEncoder.java @@ -0,0 +1,75 @@ +/* + * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.core.externalreader; + +import java.io.IOException; +import java.io.OutputStream; +import org.msgpack.core.MessagePack; +import org.msgpack.core.MessagePacker; +import org.pkl.core.externalreader.ExternalReaderMessages.*; +import org.pkl.core.messaging.BaseMessagePackEncoder; +import org.pkl.core.messaging.Message; +import org.pkl.core.messaging.ProtocolException; +import org.pkl.core.util.Nullable; + +public class ExternalReaderMessagePackEncoder extends BaseMessagePackEncoder { + + public ExternalReaderMessagePackEncoder(MessagePacker packer) { + super(packer); + } + + public ExternalReaderMessagePackEncoder(OutputStream outputStream) { + this(MessagePack.newDefaultPacker(outputStream)); + } + + @Override + protected @Nullable void encodeMessage(Message msg) throws ProtocolException, IOException { + switch (msg.type()) { + case INITIALIZE_MODULE_READER_REQUEST -> { + var m = (InitializeModuleReaderRequest) msg; + packer.packMapHeader(2); + packKeyValue("requestId", m.requestId()); + packKeyValue("scheme", m.scheme()); + } + case INITIALIZE_RESOURCE_READER_REQUEST -> { + var m = (InitializeResourceReaderRequest) msg; + packer.packMapHeader(2); + packKeyValue("requestId", m.requestId()); + packKeyValue("scheme", m.scheme()); + } + case INITIALIZE_MODULE_READER_RESPONSE -> { + var m = (InitializeModuleReaderResponse) msg; + packMapHeader(1, m.spec()); + packKeyValue("requestId", m.requestId()); + if (m.spec() != null) { + packer.packString("spec"); + packModuleReaderSpec(m.spec()); + } + } + case INITIALIZE_RESOURCE_READER_RESPONSE -> { + var m = (InitializeResourceReaderResponse) msg; + packMapHeader(1, m.spec()); + packKeyValue("requestId", m.requestId()); + if (m.spec() != null) { + packer.packString("spec"); + packResourceReaderSpec(m.spec()); + } + } + case CLOSE_EXTERNAL_PROCESS -> packer.packMapHeader(0); + default -> super.encodeMessage(msg); + } + } +} diff --git a/pkl-core/src/main/java/org/pkl/core/externalreader/ExternalReaderMessages.java b/pkl-core/src/main/java/org/pkl/core/externalreader/ExternalReaderMessages.java new file mode 100644 index 000000000..0b1a87d30 --- /dev/null +++ b/pkl-core/src/main/java/org/pkl/core/externalreader/ExternalReaderMessages.java @@ -0,0 +1,58 @@ +/* + * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.core.externalreader; + +import org.pkl.core.messaging.Message.*; +import org.pkl.core.messaging.Messages.ModuleReaderSpec; +import org.pkl.core.messaging.Messages.ResourceReaderSpec; +import org.pkl.core.util.Nullable; + +public class ExternalReaderMessages { + + public record InitializeModuleReaderRequest(long requestId, String scheme) + implements Server.Request { + public Type type() { + return Type.INITIALIZE_MODULE_READER_REQUEST; + } + } + + public record InitializeResourceReaderRequest(long requestId, String scheme) + implements Server.Request { + public Type type() { + return Type.INITIALIZE_RESOURCE_READER_REQUEST; + } + } + + public record InitializeModuleReaderResponse(long requestId, @Nullable ModuleReaderSpec spec) + implements Client.Response { + public Type type() { + return Type.INITIALIZE_MODULE_READER_RESPONSE; + } + } + + public record InitializeResourceReaderResponse(long requestId, @Nullable ResourceReaderSpec spec) + implements Client.Response { + public Type type() { + return Type.INITIALIZE_RESOURCE_READER_RESPONSE; + } + } + + public record CloseExternalProcess() implements Server.OneWay { + public Type type() { + return Type.CLOSE_EXTERNAL_PROCESS; + } + } +} diff --git a/pkl-core/src/main/java/org/pkl/core/externalreader/ExternalReaderProcess.java b/pkl-core/src/main/java/org/pkl/core/externalreader/ExternalReaderProcess.java new file mode 100644 index 000000000..f89d46313 --- /dev/null +++ b/pkl-core/src/main/java/org/pkl/core/externalreader/ExternalReaderProcess.java @@ -0,0 +1,54 @@ +/* + * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.core.externalreader; + +import java.io.IOException; +import org.pkl.core.messaging.MessageTransport; +import org.pkl.core.messaging.Messages.ModuleReaderSpec; +import org.pkl.core.messaging.Messages.ResourceReaderSpec; +import org.pkl.core.util.Nullable; + +/** An interface for interacting with external module/resource processes. */ +public interface ExternalReaderProcess extends AutoCloseable { + + /** + * Obtain the process's underlying {@link MessageTransport} for sending reader-specific message + * + *

May allocate resources upon first call, including spawning a child process. Must not be + * called after {@link ExternalReaderProcess#close} has been called. + */ + MessageTransport getTransport() throws ExternalReaderProcessException; + + /** Retrieve the spec, if available, of the process's module reader with the given scheme. */ + @Nullable + ModuleReaderSpec getModuleReaderSpec(String scheme) throws IOException; + + /** Retrieve the spec, if available, of the process's resource reader with the given scheme. */ + @Nullable + ResourceReaderSpec getResourceReaderSpec(String scheme) throws IOException; + + /** + * Close the external process, cleaning up any resources. + * + *

The {@link MessageTransport} is sent the {@link ExternalReaderMessages.CloseExternalProcess} + * message to request a graceful stop. A bespoke (empty) message type is used here instead of an + * OS mechanism like signals to avoid forcing external reader implementers needing to handle many + * OS-specific mechanisms. Implementations may then forcibly clean up resources after a timeout. + * Must be safe to call multiple times. + */ + @Override + void close(); +} diff --git a/pkl-core/src/main/java/org/pkl/core/externalreader/ExternalReaderProcessException.java b/pkl-core/src/main/java/org/pkl/core/externalreader/ExternalReaderProcessException.java new file mode 100644 index 000000000..6dbf4bf5d --- /dev/null +++ b/pkl-core/src/main/java/org/pkl/core/externalreader/ExternalReaderProcessException.java @@ -0,0 +1,26 @@ +/* + * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.core.externalreader; + +public final class ExternalReaderProcessException extends Exception { + public ExternalReaderProcessException(String msg) { + super(msg); + } + + public ExternalReaderProcessException(Throwable cause) { + super(cause); + } +} diff --git a/pkl-core/src/main/java/org/pkl/core/externalreader/ExternalReaderProcessImpl.java b/pkl-core/src/main/java/org/pkl/core/externalreader/ExternalReaderProcessImpl.java new file mode 100644 index 000000000..003bcba8d --- /dev/null +++ b/pkl-core/src/main/java/org/pkl/core/externalreader/ExternalReaderProcessImpl.java @@ -0,0 +1,226 @@ +/* + * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.core.externalreader; + +import java.io.IOException; +import java.lang.ProcessBuilder.Redirect; +import java.util.ArrayList; +import java.util.Map; +import java.util.Objects; +import java.util.Random; +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Future; +import javax.annotation.concurrent.GuardedBy; +import org.pkl.core.Duration; +import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader; +import org.pkl.core.externalreader.ExternalReaderMessages.*; +import org.pkl.core.messaging.MessageTransport; +import org.pkl.core.messaging.MessageTransports; +import org.pkl.core.messaging.Messages.ModuleReaderSpec; +import org.pkl.core.messaging.Messages.ResourceReaderSpec; +import org.pkl.core.messaging.ProtocolException; +import org.pkl.core.util.LateInit; +import org.pkl.core.util.Nullable; + +public class ExternalReaderProcessImpl implements ExternalReaderProcess { + + private static final Duration CLOSE_TIMEOUT = Duration.ofSeconds(3); + + private final ExternalReader spec; + private final @Nullable String logPrefix; + private final Map> initializeModuleReaderResponses = + new ConcurrentHashMap<>(); + private final Map> + initializeResourceReaderResponses = new ConcurrentHashMap<>(); + + private @GuardedBy("this") boolean closed = false; + + @LateInit + @GuardedBy("this") + private Process process; + + @LateInit + @GuardedBy("this") + private MessageTransport transport; + + private void log(String msg) { + if (logPrefix != null) { + System.err.println(logPrefix + msg); + } + } + + public ExternalReaderProcessImpl(ExternalReader spec) { + this.spec = spec; + logPrefix = + Objects.equals(System.getenv("PKL_DEBUG"), "1") + ? "[pkl-core][external-process][" + spec.executable() + "] " + : null; + } + + @Override + public synchronized MessageTransport getTransport() throws ExternalReaderProcessException { + if (closed) { + throw new ExternalReaderProcessException("ExternalProcessImpl has already been closed"); + } + if (process != null) { + if (!process.isAlive()) { + throw new ExternalReaderProcessException("ExternalProcessImpl process is no longer alive"); + } + + return transport; + } + + // This relies on Java/OS behavior around PATH resolution, absolute/relative paths, etc. + var command = new ArrayList(); + command.add(spec.executable()); + command.addAll(spec.arguments()); + + var builder = new ProcessBuilder(command); + builder.redirectError(Redirect.INHERIT); // inherit stderr from this pkl process + try { + process = builder.start(); + } catch (IOException e) { + throw new ExternalReaderProcessException(e); + } + transport = + MessageTransports.stream( + new ExternalReaderMessagePackDecoder(process.getInputStream()), + new ExternalReaderMessagePackEncoder(process.getOutputStream()), + this::log); + + var rxThread = new Thread(this::runTransport, "ExternalProcessImpl rxThread for " + spec); + rxThread.setDaemon(true); + rxThread.start(); + + return transport; + } + + /** + * Runs the underlying message transport so it can receive responses from the child process. + * + *

Blocks until the underlying transport is closed. + */ + private void runTransport() { + try { + transport.start( + (msg) -> { + throw new ProtocolException("Unexpected incoming one-way message: " + msg); + }, + (msg) -> { + throw new ProtocolException("Unexpected incoming request message: " + msg); + }); + } catch (ProtocolException | IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public synchronized void close() { + closed = true; + if (process == null || !process.isAlive()) { + return; + } + + try { + if (transport != null) { + transport.send(new CloseExternalProcess()); + transport.close(); + } + + // forcefully stop the process after the timeout + // note that both transport.close() and process.destroy() are safe to call multiple times + new Timer() + .schedule( + new TimerTask() { + @Override + public void run() { + if (process != null) { + transport.close(); + process.destroyForcibly(); + } + } + }, + CLOSE_TIMEOUT.inWholeMillis()); + + // block on process exit + process.onExit().get(); + } catch (Exception e) { + transport.close(); + process.destroyForcibly(); + } finally { + process = null; + transport = null; + } + } + + @Override + public @Nullable ModuleReaderSpec getModuleReaderSpec(String uriScheme) throws IOException { + return MessageTransports.resolveFuture( + initializeModuleReaderResponses.computeIfAbsent( + uriScheme, + (scheme) -> { + var future = new CompletableFuture<@Nullable ModuleReaderSpec>(); + var request = new InitializeModuleReaderRequest(new Random().nextLong(), scheme); + try { + getTransport() + .send( + request, + (response) -> { + if (response instanceof InitializeModuleReaderResponse resp) { + future.complete(resp.spec()); + } else { + future.completeExceptionally( + new ProtocolException("unexpected response")); + } + }); + } catch (ProtocolException | IOException | ExternalReaderProcessException e) { + future.completeExceptionally(e); + } + return future; + })); + } + + @Override + public @Nullable ResourceReaderSpec getResourceReaderSpec(String uriScheme) throws IOException { + return MessageTransports.resolveFuture( + initializeResourceReaderResponses.computeIfAbsent( + uriScheme, + (scheme) -> { + var future = new CompletableFuture<@Nullable ResourceReaderSpec>(); + var request = new InitializeResourceReaderRequest(new Random().nextLong(), scheme); + try { + getTransport() + .send( + request, + (response) -> { + log(response.toString()); + if (response instanceof InitializeResourceReaderResponse resp) { + future.complete(resp.spec()); + } else { + future.completeExceptionally( + new ProtocolException("unexpected response")); + } + }); + } catch (ProtocolException | IOException | ExternalReaderProcessException e) { + future.completeExceptionally(e); + } + return future; + })); + } +} diff --git a/pkl-core/src/main/java/org/pkl/core/externalreader/package-info.java b/pkl-core/src/main/java/org/pkl/core/externalreader/package-info.java new file mode 100644 index 000000000..735a048d5 --- /dev/null +++ b/pkl-core/src/main/java/org/pkl/core/externalreader/package-info.java @@ -0,0 +1,4 @@ +@NonnullByDefault +package org.pkl.core.externalreader; + +import org.pkl.core.util.NonnullByDefault; diff --git a/pkl-core/src/main/java/org/pkl/core/messaging/AbstractMessagePackDecoder.java b/pkl-core/src/main/java/org/pkl/core/messaging/AbstractMessagePackDecoder.java new file mode 100644 index 000000000..5a3c3fe94 --- /dev/null +++ b/pkl-core/src/main/java/org/pkl/core/messaging/AbstractMessagePackDecoder.java @@ -0,0 +1,220 @@ +/* + * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.core.messaging; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.msgpack.core.MessagePack; +import org.msgpack.core.MessageTypeException; +import org.msgpack.core.MessageUnpacker; +import org.msgpack.value.Value; +import org.msgpack.value.impl.ImmutableStringValueImpl; +import org.pkl.core.messaging.Message.Type; +import org.pkl.core.util.ErrorMessages; +import org.pkl.core.util.Nullable; + +public abstract class AbstractMessagePackDecoder implements MessageDecoder { + + protected final MessageUnpacker unpacker; + + public AbstractMessagePackDecoder(MessageUnpacker unpacker) { + this.unpacker = unpacker; + } + + public AbstractMessagePackDecoder(InputStream stream) { + this(MessagePack.newDefaultUnpacker(stream)); + } + + protected abstract @Nullable Message decodeMessage(Type msgType, Map map) + throws DecodeException, URISyntaxException; + + @Override + public @Nullable Message decode() throws IOException, DecodeException { + if (!unpacker.hasNext()) { + return null; + } + + int code; + try { + var arraySize = unpacker.unpackArrayHeader(); + if (arraySize != 2) { + throw new DecodeException(ErrorMessages.create("malformedMessageHeaderLength", arraySize)); + } + code = unpacker.unpackInt(); + } catch (MessageTypeException e) { + throw new DecodeException(ErrorMessages.create("malformedMessageHeaderException"), e); + } + + Type msgType; + try { + msgType = Type.fromInt(code); + } catch (IllegalArgumentException e) { + throw new DecodeException( + ErrorMessages.create("malformedMessageHeaderUnrecognizedCode", Integer.toHexString(code)), + e); + } + + try { + var map = unpacker.unpackValue().asMapValue().map(); + var decoded = decodeMessage(msgType, map); + if (decoded != null) { + return decoded; + } + throw new DecodeException( + ErrorMessages.create("unhandledMessageCode", Integer.toHexString(code))); + } catch (MessageTypeException | URISyntaxException e) { + throw new DecodeException(ErrorMessages.create("malformedMessageBody", code), e); + } + } + + protected static @Nullable Value getNullable(Map map, String key) { + return map.get(new ImmutableStringValueImpl(key)); + } + + protected static Value get(Map map, String key) throws DecodeException { + var value = map.get(new ImmutableStringValueImpl(key)); + if (value == null) { + throw new DecodeException(ErrorMessages.create("missingMessageParameter", key)); + } + return value; + } + + protected static String unpackString(Map map, String key) throws DecodeException { + return get(map, key).asStringValue().asString(); + } + + protected static @Nullable String unpackStringOrNull(Map map, String key) { + var value = getNullable(map, key); + if (value == null) { + return null; + } + return value.asStringValue().asString(); + } + + protected static @Nullable T unpackStringOrNull( + Map map, String key, Function mapper) { + var value = getNullable(map, key); + if (value == null) { + return null; + } + return mapper.apply(value.asStringValue().asString()); + } + + protected static byte @Nullable [] unpackByteArray(Map map, String key) { + var value = getNullable(map, key); + if (value == null) { + return null; + } + return value.asBinaryValue().asByteArray(); + } + + protected static boolean unpackBoolean(Map map, String key) throws DecodeException { + return get(map, key).asBooleanValue().getBoolean(); + } + + protected static int unpackInt(Map map, String key) throws DecodeException { + return get(map, key).asIntegerValue().asInt(); + } + + protected static long unpackLong(Map map, String key) throws DecodeException { + return get(map, key).asIntegerValue().asLong(); + } + + protected static @Nullable Long unpackLongOrNull(Map map, String key) { + var value = getNullable(map, key); + if (value == null) { + return null; + } + return value.asIntegerValue().asLong(); + } + + protected static @Nullable T unpackLongOrNull( + Map map, String key, Function mapper) { + var value = unpackLongOrNull(map, key); + if (value == null) { + return null; + } + return mapper.apply(value); + } + + protected static @Nullable List unpackStringListOrNull( + Map map, String key) { + var value = getNullable(map, key); + if (value == null) { + return null; + } + + return value.asArrayValue().list().stream().map((it) -> it.asStringValue().asString()).toList(); + } + + protected static @Nullable Map unpackStringMapOrNull( + Map map, String key) { + var value = getNullable(map, key); + if (value == null) { + return null; + } + + return value.asMapValue().entrySet().stream() + .collect( + Collectors.toMap( + (e) -> e.getKey().asStringValue().asString(), + (e) -> e.getValue().asStringValue().asString())); + } + + protected static @Nullable List unpackStringListOrNull( + Map map, String key, Function mapper) { + var value = unpackStringListOrNull(map, key); + if (value == null) { + return null; + } + + return value.stream().map(mapper).toList(); + } + + protected static @Nullable List unpackListOrNull( + Map map, String key, Function mapper) { + var keys = getNullable(map, key); + if (keys == null) { + return null; + } + + var result = new ArrayList(keys.asArrayValue().size()); + for (Value value : keys.asArrayValue()) { + result.add(mapper.apply(value)); + } + return result; + } + + protected static @Nullable Map unpackStringMapOrNull( + Map map, String key, Function, T> mapper) { + var value = getNullable(map, key); + if (value == null) { + return null; + } + + return value.asMapValue().entrySet().stream() + .collect( + Collectors.toMap( + (e) -> e.getKey().asStringValue().asString(), + (e) -> mapper.apply(e.getValue().asMapValue().map()))); + } +} diff --git a/pkl-core/src/main/java/org/pkl/core/messaging/AbstractMessagePackEncoder.java b/pkl-core/src/main/java/org/pkl/core/messaging/AbstractMessagePackEncoder.java new file mode 100644 index 000000000..841ff50ba --- /dev/null +++ b/pkl-core/src/main/java/org/pkl/core/messaging/AbstractMessagePackEncoder.java @@ -0,0 +1,184 @@ +/* + * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.core.messaging; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.Collection; +import java.util.Map; +import java.util.function.Function; +import org.msgpack.core.MessagePack; +import org.msgpack.core.MessagePacker; +import org.pkl.core.util.Nullable; + +public abstract class AbstractMessagePackEncoder implements MessageEncoder { + + protected final MessagePacker packer; + + public AbstractMessagePackEncoder(MessagePacker packer) { + this.packer = packer; + } + + public AbstractMessagePackEncoder(OutputStream stream) { + this(MessagePack.newDefaultPacker(stream)); + } + + protected abstract @Nullable void encodeMessage(Message msg) + throws ProtocolException, IOException; + + @Override + public final void encode(Message msg) throws IOException, ProtocolException { + packer.packArrayHeader(2); + packer.packInt(msg.type().getCode()); + encodeMessage(msg); + packer.flush(); + } + + protected void packMapHeader(int size, @Nullable Object value1) throws IOException { + packer.packMapHeader(size + (value1 != null ? 1 : 0)); + } + + protected void packMapHeader(int size, @Nullable Object value1, @Nullable Object value2) + throws IOException { + packer.packMapHeader(size + (value1 != null ? 1 : 0) + (value2 != null ? 1 : 0)); + } + + protected void packMapHeader( + int size, + @Nullable Object value1, + @Nullable Object value2, + @Nullable Object value3, + @Nullable Object value4, + @Nullable Object value5, + @Nullable Object value6, + @Nullable Object value7, + @Nullable Object value8, + @Nullable Object value9, + @Nullable Object valueA, + @Nullable Object valueB, + @Nullable Object valueC, + @Nullable Object valueD, + @Nullable Object valueE, + @Nullable Object valueF) + throws IOException { + packer.packMapHeader( + size + + (value1 != null ? 1 : 0) + + (value2 != null ? 1 : 0) + + (value3 != null ? 1 : 0) + + (value4 != null ? 1 : 0) + + (value5 != null ? 1 : 0) + + (value6 != null ? 1 : 0) + + (value7 != null ? 1 : 0) + + (value8 != null ? 1 : 0) + + (value9 != null ? 1 : 0) + + (valueA != null ? 1 : 0) + + (valueB != null ? 1 : 0) + + (valueC != null ? 1 : 0) + + (valueD != null ? 1 : 0) + + (valueE != null ? 1 : 0) + + (valueF != null ? 1 : 0)); + } + + protected void packKeyValue(String name, @Nullable Integer value) throws IOException { + if (value == null) { + return; + } + packer.packString(name); + packer.packInt(value); + } + + protected void packKeyValue(String name, @Nullable Long value) throws IOException { + if (value == null) { + return; + } + packer.packString(name); + packer.packLong(value); + } + + protected void packKeyValueLong(String name, @Nullable T value, Function mapper) + throws IOException { + if (value == null) { + return; + } + packKeyValue(name, mapper.apply(value)); + } + + protected void packKeyValue(String name, @Nullable String value) throws IOException { + if (value == null) { + return; + } + packer.packString(name); + packer.packString(value); + } + + protected void packKeyValueString(String name, @Nullable T value, Function mapper) + throws IOException { + if (value == null) { + return; + } + packKeyValue(name, mapper.apply(value)); + } + + protected void packKeyValue(String name, @Nullable Collection value) throws IOException { + if (value == null) { + return; + } + packer.packString(name); + packer.packArrayHeader(value.size()); + for (String elem : value) { + packer.packString(elem); + } + } + + protected void packKeyValue( + String name, @Nullable Collection value, Function mapper) throws IOException { + if (value == null) { + return; + } + packer.packString(name); + packer.packArrayHeader(value.size()); + for (T elem : value) { + packer.packString(mapper.apply(elem)); + } + } + + protected void packKeyValue(String name, @Nullable Map value) throws IOException { + if (value == null) { + return; + } + packer.packString(name); + packer.packMapHeader(value.size()); + for (Map.Entry e : value.entrySet()) { + packer.packString(e.getKey()); + packer.packString(e.getValue()); + } + } + + protected void packKeyValue(String name, byte @Nullable [] value) throws IOException { + if (value == null) { + return; + } + packer.packString(name); + packer.packBinaryHeader(value.length); + packer.writePayload(value); + } + + protected void packKeyValue(String name, boolean value) throws IOException { + packer.packString(name); + packer.packBoolean(value); + } +} diff --git a/pkl-core/src/main/java/org/pkl/core/messaging/BaseMessagePackDecoder.java b/pkl-core/src/main/java/org/pkl/core/messaging/BaseMessagePackDecoder.java new file mode 100644 index 000000000..c4fa6ddee --- /dev/null +++ b/pkl-core/src/main/java/org/pkl/core/messaging/BaseMessagePackDecoder.java @@ -0,0 +1,131 @@ +/* + * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.core.messaging; + +import java.io.InputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.*; +import org.msgpack.core.MessageUnpacker; +import org.msgpack.value.Value; +import org.pkl.core.messaging.Message.Type; +import org.pkl.core.messaging.Messages.*; +import org.pkl.core.module.PathElement; +import org.pkl.core.util.Nullable; + +public class BaseMessagePackDecoder extends AbstractMessagePackDecoder { + + public BaseMessagePackDecoder(MessageUnpacker unpacker) { + super(unpacker); + } + + public BaseMessagePackDecoder(InputStream stream) { + super(stream); + } + + protected @Nullable Message decodeMessage(Type msgType, Map map) + throws DecodeException, URISyntaxException { + return switch (msgType) { + case READ_RESOURCE_REQUEST -> + new ReadResourceRequest( + unpackLong(map, "requestId"), + unpackLong(map, "evaluatorId"), + new URI(unpackString(map, "uri"))); + case READ_RESOURCE_RESPONSE -> + new ReadResourceResponse( + unpackLong(map, "requestId"), + unpackLong(map, "evaluatorId"), + unpackByteArray(map, "contents"), + unpackStringOrNull(map, "error")); + case READ_MODULE_REQUEST -> + new ReadModuleRequest( + unpackLong(map, "requestId"), + unpackLong(map, "evaluatorId"), + new URI(unpackString(map, "uri"))); + case READ_MODULE_RESPONSE -> + new ReadModuleResponse( + unpackLong(map, "requestId"), + unpackLong(map, "evaluatorId"), + unpackStringOrNull(map, "contents"), + unpackStringOrNull(map, "error")); + case LIST_RESOURCES_REQUEST -> + new ListResourcesRequest( + unpackLong(map, "requestId"), + unpackLong(map, "evaluatorId"), + new URI(unpackString(map, "uri"))); + case LIST_RESOURCES_RESPONSE -> + new ListResourcesResponse( + unpackLong(map, "requestId"), + unpackLong(map, "evaluatorId"), + unpackPathElements(map, "pathElements"), + unpackStringOrNull(map, "error")); + case LIST_MODULES_REQUEST -> + new ListModulesRequest( + unpackLong(map, "requestId"), + unpackLong(map, "evaluatorId"), + new URI(unpackString(map, "uri"))); + case LIST_MODULES_RESPONSE -> + new ListModulesResponse( + unpackLong(map, "requestId"), + unpackLong(map, "evaluatorId"), + unpackPathElements(map, "pathElements"), + unpackStringOrNull(map, "error")); + default -> null; + }; + } + + protected static @Nullable ModuleReaderSpec unpackModuleReaderSpec(@Nullable Value value) + throws DecodeException { + if (value == null) { + return null; + } + var map = value.asMapValue().map(); + return new ModuleReaderSpec( + unpackString(map, "scheme"), + unpackBoolean(map, "hasHierarchicalUris"), + unpackBoolean(map, "isLocal"), + unpackBoolean(map, "isGlobbable")); + } + + protected static @Nullable ResourceReaderSpec unpackResourceReaderSpec(@Nullable Value value) + throws DecodeException { + if (value == null) { + return null; + } + var map = value.asMapValue().map(); + return new ResourceReaderSpec( + unpackString(map, "scheme"), + unpackBoolean(map, "hasHierarchicalUris"), + unpackBoolean(map, "isGlobbable")); + } + + protected static @Nullable List unpackPathElements(Map map, String key) + throws DecodeException { + var value = getNullable(map, key); + if (value == null) { + return null; + } + + var result = new ArrayList(value.asArrayValue().size()); + for (Value pathElement : value.asArrayValue()) { + var pathElementMap = pathElement.asMapValue().map(); + result.add( + new PathElement( + unpackString(pathElementMap, "name"), unpackBoolean(pathElementMap, "isDirectory"))); + } + return result; + } +} diff --git a/pkl-core/src/main/java/org/pkl/core/messaging/BaseMessagePackEncoder.java b/pkl-core/src/main/java/org/pkl/core/messaging/BaseMessagePackEncoder.java new file mode 100644 index 000000000..0b7aaccad --- /dev/null +++ b/pkl-core/src/main/java/org/pkl/core/messaging/BaseMessagePackEncoder.java @@ -0,0 +1,136 @@ +/* + * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.core.messaging; + +import java.io.IOException; +import java.io.OutputStream; +import org.msgpack.core.MessagePacker; +import org.pkl.core.messaging.Messages.*; +import org.pkl.core.module.PathElement; +import org.pkl.core.util.ErrorMessages; +import org.pkl.core.util.Nullable; + +public class BaseMessagePackEncoder extends AbstractMessagePackEncoder { + + public BaseMessagePackEncoder(MessagePacker packer) { + super(packer); + } + + public BaseMessagePackEncoder(OutputStream stream) { + super(stream); + } + + protected void packModuleReaderSpec(ModuleReaderSpec reader) throws IOException { + packer.packMapHeader(4); + packKeyValue("scheme", reader.scheme()); + packKeyValue("hasHierarchicalUris", reader.hasHierarchicalUris()); + packKeyValue("isLocal", reader.isLocal()); + packKeyValue("isGlobbable", reader.isGlobbable()); + } + + protected void packResourceReaderSpec(ResourceReaderSpec reader) throws IOException { + packer.packMapHeader(3); + packKeyValue("scheme", reader.scheme()); + packKeyValue("hasHierarchicalUris", reader.hasHierarchicalUris()); + packKeyValue("isGlobbable", reader.isGlobbable()); + } + + protected void packPathElement(PathElement pathElement) throws IOException { + packer.packMapHeader(2); + packKeyValue("name", pathElement.getName()); + packKeyValue("isDirectory", pathElement.isDirectory()); + } + + protected @Nullable void encodeMessage(Message msg) throws ProtocolException, IOException { + switch (msg.type()) { + case READ_RESOURCE_REQUEST -> { + var m = (ReadResourceRequest) msg; + packer.packMapHeader(3); + packKeyValue("requestId", m.requestId()); + packKeyValue("evaluatorId", m.evaluatorId()); + packKeyValue("uri", m.uri().toString()); + } + case READ_RESOURCE_RESPONSE -> { + var m = (ReadResourceResponse) msg; + packMapHeader(2, m.contents(), m.error()); + packKeyValue("requestId", m.requestId()); + packKeyValue("evaluatorId", m.evaluatorId()); + packKeyValue("contents", m.contents()); + packKeyValue("error", m.error()); + } + case READ_MODULE_REQUEST -> { + var m = (ReadModuleRequest) msg; + packer.packMapHeader(3); + packKeyValue("requestId", m.requestId()); + packKeyValue("evaluatorId", m.evaluatorId()); + packKeyValue("uri", m.uri().toString()); + } + case READ_MODULE_RESPONSE -> { + var m = (ReadModuleResponse) msg; + packMapHeader(2, m.contents(), m.error()); + packKeyValue("requestId", m.requestId()); + packKeyValue("evaluatorId", m.evaluatorId()); + packKeyValue("contents", m.contents()); + packKeyValue("error", m.error()); + } + case LIST_RESOURCES_REQUEST -> { + var m = (ListResourcesRequest) msg; + packer.packMapHeader(3); + packKeyValue("requestId", m.requestId()); + packKeyValue("evaluatorId", m.evaluatorId()); + packKeyValue("uri", m.uri().toString()); + } + case LIST_RESOURCES_RESPONSE -> { + var m = (ListResourcesResponse) msg; + packMapHeader(2, m.pathElements(), m.error()); + packKeyValue("requestId", m.requestId()); + packKeyValue("evaluatorId", m.evaluatorId()); + if (m.pathElements() != null) { + packer.packString("pathElements"); + packer.packArrayHeader(m.pathElements().size()); + for (var pathElement : m.pathElements()) { + packPathElement(pathElement); + } + } + packKeyValue("error", m.error()); + } + case LIST_MODULES_REQUEST -> { + var m = (ListModulesRequest) msg; + packer.packMapHeader(3); + packKeyValue("requestId", m.requestId()); + packKeyValue("evaluatorId", m.evaluatorId()); + packKeyValue("uri", m.uri().toString()); + } + case LIST_MODULES_RESPONSE -> { + var m = (ListModulesResponse) msg; + packMapHeader(2, m.pathElements(), m.error()); + packKeyValue("requestId", m.requestId()); + packKeyValue("evaluatorId", m.evaluatorId()); + if (m.pathElements() != null) { + packer.packString("pathElements"); + packer.packArrayHeader(m.pathElements().size()); + for (var pathElement : m.pathElements()) { + packPathElement(pathElement); + } + } + packKeyValue("error", m.error()); + } + default -> + throw new ProtocolException( + ErrorMessages.create("unhandledMessageType", msg.type().toString())); + } + } +} diff --git a/pkl-server/src/main/kotlin/org/pkl/server/ServerException.kt b/pkl-core/src/main/java/org/pkl/core/messaging/DecodeException.java similarity index 62% rename from pkl-server/src/main/kotlin/org/pkl/server/ServerException.kt rename to pkl-core/src/main/java/org/pkl/core/messaging/DecodeException.java index 8b829f81b..75356ee93 100644 --- a/pkl-server/src/main/kotlin/org/pkl/server/ServerException.kt +++ b/pkl-core/src/main/java/org/pkl/core/messaging/DecodeException.java @@ -13,12 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.pkl.server +package org.pkl.core.messaging; -sealed class ServerException(msg: String, cause: Throwable?) : Exception(msg, cause) +public final class DecodeException extends ProtocolException { -open class ProtocolException(msg: String, cause: Throwable? = null) : ServerException(msg, cause) + public DecodeException(String msg, Throwable cause) { + super(msg, cause); + } -class InvalidCommandException(msg: String, cause: Throwable? = null) : ServerException(msg, cause) - -class DecodeException(msg: String, cause: Throwable? = null) : ProtocolException(msg, cause) + public DecodeException(String msg) { + super(msg); + } +} diff --git a/pkl-core/src/main/java/org/pkl/core/messaging/Message.java b/pkl-core/src/main/java/org/pkl/core/messaging/Message.java new file mode 100644 index 000000000..f8ef4f6ae --- /dev/null +++ b/pkl-core/src/main/java/org/pkl/core/messaging/Message.java @@ -0,0 +1,89 @@ +/* + * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.core.messaging; + +public interface Message { + + Type type(); + + enum Type { + CREATE_EVALUATOR_REQUEST(0x20), + CREATE_EVALUATOR_RESPONSE(0x21), + CLOSE_EVALUATOR(0x22), + EVALUATE_REQUEST(0x23), + EVALUATE_RESPONSE(0x24), + LOG_MESSAGE(0x25), + READ_RESOURCE_REQUEST(0x26), + READ_RESOURCE_RESPONSE(0x27), + READ_MODULE_REQUEST(0x28), + READ_MODULE_RESPONSE(0x29), + LIST_RESOURCES_REQUEST(0x2a), + LIST_RESOURCES_RESPONSE(0x2b), + LIST_MODULES_REQUEST(0x2c), + LIST_MODULES_RESPONSE(0x2d), + INITIALIZE_MODULE_READER_REQUEST(0x2e), + INITIALIZE_MODULE_READER_RESPONSE(0x2f), + INITIALIZE_RESOURCE_READER_REQUEST(0x30), + INITIALIZE_RESOURCE_READER_RESPONSE(0x31), + CLOSE_EXTERNAL_PROCESS(0x32); + + private final int code; + + Type(int code) { + this.code = code; + } + + public static Type fromInt(int val) throws IllegalArgumentException { + for (Type t : Type.values()) { + if (t.code == val) { + return t; + } + } + + throw new IllegalArgumentException("Unknown Message.Type code"); + } + + public int getCode() { + return code; + } + } + + interface OneWay extends Message {} + + interface Request extends Message { + long requestId(); + } + + interface Response extends Message { + long requestId(); + } + + interface Client extends Message { + interface Request extends Client, Message.Request {} + + interface Response extends Client, Message.Response {} + + interface OneWay extends Client, Message.OneWay {} + } + + interface Server extends Message { + interface Request extends Server, Message.Request {} + + interface Response extends Server, Message.Response {} + + interface OneWay extends Server, Message.OneWay {} + } +} diff --git a/pkl-server/src/main/kotlin/org/pkl/server/MessageDecoder.kt b/pkl-core/src/main/java/org/pkl/core/messaging/MessageDecoder.java similarity index 77% rename from pkl-server/src/main/kotlin/org/pkl/server/MessageDecoder.kt rename to pkl-core/src/main/java/org/pkl/core/messaging/MessageDecoder.java index 7dc29a0e9..f604fb9b7 100644 --- a/pkl-server/src/main/kotlin/org/pkl/server/MessageDecoder.kt +++ b/pkl-core/src/main/java/org/pkl/core/messaging/MessageDecoder.java @@ -13,9 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.pkl.server +package org.pkl.core.messaging; + +import java.io.IOException; +import org.pkl.core.util.Nullable; /** Decodes a stream of messages. */ -internal interface MessageDecoder { - fun decode(): Message? +public interface MessageDecoder { + @Nullable + Message decode() throws IOException, DecodeException; } diff --git a/pkl-server/src/main/kotlin/org/pkl/server/MessageEncoder.kt b/pkl-core/src/main/java/org/pkl/core/messaging/MessageEncoder.java similarity index 81% rename from pkl-server/src/main/kotlin/org/pkl/server/MessageEncoder.kt rename to pkl-core/src/main/java/org/pkl/core/messaging/MessageEncoder.java index 7adda9a9d..1d68b9c2b 100644 --- a/pkl-server/src/main/kotlin/org/pkl/server/MessageEncoder.kt +++ b/pkl-core/src/main/java/org/pkl/core/messaging/MessageEncoder.java @@ -13,9 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.pkl.server +package org.pkl.core.messaging; + +import java.io.IOException; /** Encodes a stream of messages. */ -internal interface MessageEncoder { - fun encode(msg: Message) +public interface MessageEncoder { + void encode(Message msg) throws IOException, ProtocolException; } diff --git a/pkl-core/src/main/java/org/pkl/core/messaging/MessageTransport.java b/pkl-core/src/main/java/org/pkl/core/messaging/MessageTransport.java new file mode 100644 index 000000000..f94155573 --- /dev/null +++ b/pkl-core/src/main/java/org/pkl/core/messaging/MessageTransport.java @@ -0,0 +1,46 @@ +/* + * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.core.messaging; + +import java.io.IOException; + +/** A bidirectional transport for sending and receiving messages. */ +public interface MessageTransport extends AutoCloseable { + interface OneWayHandler { + void handleOneWay(Message.OneWay msg) throws ProtocolException; + } + + interface RequestHandler { + void handleRequest(Message.Request msg) throws ProtocolException, IOException; + } + + interface ResponseHandler { + void handleResponse(Message.Response msg) throws ProtocolException; + } + + void start(OneWayHandler oneWayHandler, RequestHandler requestHandler) + throws ProtocolException, IOException; + + void send(Message.OneWay message) throws ProtocolException, IOException; + + void send(Message.Request message, ResponseHandler responseHandler) + throws ProtocolException, IOException; + + void send(Message.Response message) throws ProtocolException, IOException; + + @Override + void close(); +} diff --git a/pkl-core/src/main/java/org/pkl/core/messaging/MessageTransports.java b/pkl-core/src/main/java/org/pkl/core/messaging/MessageTransports.java new file mode 100644 index 000000000..ee5b3c662 --- /dev/null +++ b/pkl-core/src/main/java/org/pkl/core/messaging/MessageTransports.java @@ -0,0 +1,197 @@ +/* + * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.core.messaging; + +import java.io.IOException; +import java.text.MessageFormat; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import org.pkl.core.messaging.Message.OneWay; +import org.pkl.core.messaging.Message.Response; +import org.pkl.core.util.ErrorMessages; +import org.pkl.core.util.Pair; + +/** Factory methods for creating [MessageTransport]s. */ +public class MessageTransports { + + public interface Logger { + void log(String msg); + } + + /** Creates a message transport that reads from [inputStream] and writes to [outputStream]. */ + public static MessageTransport stream( + MessageDecoder decoder, MessageEncoder encoder, Logger logger) { + return new EncodingMessageTransport(decoder, encoder, logger); + } + + /** Creates "client" and "server" transports that are directly connected to each other. */ + public static Pair direct(Logger logger) { + var transport1 = new DirectMessageTransport(logger); + var transport2 = new DirectMessageTransport(logger); + transport1.setOther(transport2); + transport2.setOther(transport1); + return Pair.of(transport1, transport2); + } + + public static T resolveFuture(Future future) throws IOException { + try { + return future.get(); + } catch (ExecutionException | InterruptedException e) { + if (e.getCause() instanceof IOException ioExc) { + throw ioExc; + } else { + throw new IOException("external read failure: " + e.getMessage(), e.getCause()); + } + } + } + + protected static class EncodingMessageTransport extends AbstractMessageTransport { + + private final MessageDecoder decoder; + private final MessageEncoder encoder; + private volatile boolean isClosed = false; + + protected EncodingMessageTransport( + MessageDecoder decoder, MessageEncoder encoder, Logger logger) { + super(logger); + this.decoder = decoder; + this.encoder = encoder; + } + + @Override + protected void doStart() throws ProtocolException, IOException { + while (!isClosed) { + var message = decoder.decode(); + if (message == null) { + return; + } + accept(message); + } + } + + @Override + protected void doClose() { + isClosed = true; + } + + @Override + protected void doSend(Message message) throws ProtocolException, IOException { + encoder.encode(message); + } + } + + protected static class DirectMessageTransport extends AbstractMessageTransport { + + private DirectMessageTransport other; + + protected DirectMessageTransport(Logger logger) { + super(logger); + } + + @Override + protected void doStart() {} + + @Override + protected void doClose() {} + + @Override + protected void doSend(Message message) throws ProtocolException, IOException { + other.accept(message); + } + + public void setOther(DirectMessageTransport other) { + this.other = other; + } + } + + protected abstract static class AbstractMessageTransport implements MessageTransport { + + private final Logger logger; + private MessageTransport.OneWayHandler oneWayHandler; + private MessageTransport.RequestHandler requestHandler; + private final Map responseHandlers = new ConcurrentHashMap<>(); + + protected AbstractMessageTransport(Logger logger) { + this.logger = logger; + } + + protected void log(String message, Object... args) { + var formatter = new MessageFormat(message); + logger.log(formatter.format(args)); + } + + protected abstract void doStart() throws ProtocolException, IOException; + + protected abstract void doClose(); + + protected abstract void doSend(Message message) throws ProtocolException, IOException; + + protected void accept(Message message) throws ProtocolException, IOException { + log("Received message: {0}", message); + if (message instanceof Message.OneWay msg) { + oneWayHandler.handleOneWay(msg); + } else if (message instanceof Message.Request msg) { + requestHandler.handleRequest(msg); + } else if (message instanceof Message.Response msg) { + var handler = responseHandlers.remove(msg.requestId()); + if (handler == null) { + throw new ProtocolException( + ErrorMessages.create( + "unknownRequestId", message.getClass().getSimpleName(), msg.requestId())); + } + handler.handleResponse(msg); + } + } + + @Override + public final void start(OneWayHandler oneWayHandler, RequestHandler requestHandler) + throws ProtocolException, IOException { + log("Starting transport: {0}", this); + this.oneWayHandler = oneWayHandler; + this.requestHandler = requestHandler; + doStart(); + } + + @Override + public final void close() { + log("Closing transport: {0}", this); + doClose(); + responseHandlers.clear(); + } + + @Override + public void send(OneWay message) throws ProtocolException, IOException { + log("Sending message: {0}", message); + doSend(message); + } + + @Override + public void send(Message.Request message, ResponseHandler responseHandler) + throws ProtocolException, IOException { + log("Sending message: {0}", message); + responseHandlers.put(message.requestId(), responseHandler); + doSend(message); + } + + @Override + public void send(Response message) throws ProtocolException, IOException { + log("Sending message: {0}", message); + doSend(message); + } + } +} diff --git a/pkl-core/src/main/java/org/pkl/core/messaging/Messages.java b/pkl-core/src/main/java/org/pkl/core/messaging/Messages.java new file mode 100644 index 000000000..95ff1f131 --- /dev/null +++ b/pkl-core/src/main/java/org/pkl/core/messaging/Messages.java @@ -0,0 +1,126 @@ +/* + * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.core.messaging; + +import java.net.URI; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import org.pkl.core.messaging.Message.*; +import org.pkl.core.module.PathElement; +import org.pkl.core.util.Nullable; + +public class Messages { + + public record ModuleReaderSpec( + String scheme, boolean hasHierarchicalUris, boolean isLocal, boolean isGlobbable) {} + + public record ResourceReaderSpec( + String scheme, boolean hasHierarchicalUris, boolean isGlobbable) {} + + public record ListResourcesRequest(long requestId, long evaluatorId, URI uri) + implements Server.Request { + public Type type() { + return Type.LIST_RESOURCES_REQUEST; + } + } + + public record ListResourcesResponse( + long requestId, + long evaluatorId, + @Nullable List pathElements, + @Nullable String error) + implements Client.Response { + public Type type() { + return Type.LIST_RESOURCES_RESPONSE; + } + } + + public record ListModulesRequest(long requestId, long evaluatorId, URI uri) + implements Server.Request { + public Type type() { + return Type.LIST_MODULES_REQUEST; + } + } + + public record ListModulesResponse( + long requestId, + long evaluatorId, + @Nullable List pathElements, + @Nullable String error) + implements Client.Response { + public Type type() { + return Type.LIST_MODULES_RESPONSE; + } + } + + public record ReadResourceRequest(long requestId, long evaluatorId, URI uri) + implements Message.Request { + public Type type() { + return Type.READ_RESOURCE_REQUEST; + } + } + + public record ReadResourceResponse( + long requestId, long evaluatorId, byte @Nullable [] contents, @Nullable String error) + implements Client.Response { + + // workaround for kotlin bridging issue where `byte @Nullable [] contents` isn't detected as + // nullable + // public ReadResourceResponse(long requestId, long evaluatorId, @Nullable String error) { + // this(requestId, evaluatorId, null, error); + // } + + public Type type() { + return Type.READ_RESOURCE_RESPONSE; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ReadResourceResponse that = (ReadResourceResponse) o; + return requestId == that.requestId + && evaluatorId == that.evaluatorId + && Objects.equals(error, that.error) + && Arrays.equals(contents, that.contents); + } + + @Override + public int hashCode() { + return Objects.hash(requestId, evaluatorId, Arrays.hashCode(contents), error); + } + } + + public record ReadModuleRequest(long requestId, long evaluatorId, URI uri) + implements Message.Request { + public Type type() { + return Type.READ_MODULE_REQUEST; + } + } + + public record ReadModuleResponse( + long requestId, long evaluatorId, @Nullable String contents, @Nullable String error) + implements Client.Response { + public Type type() { + return Type.READ_MODULE_RESPONSE; + } + } +} diff --git a/pkl-core/src/main/java/org/pkl/core/messaging/ProtocolException.java b/pkl-core/src/main/java/org/pkl/core/messaging/ProtocolException.java new file mode 100644 index 000000000..da4459138 --- /dev/null +++ b/pkl-core/src/main/java/org/pkl/core/messaging/ProtocolException.java @@ -0,0 +1,26 @@ +/* + * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.core.messaging; + +public class ProtocolException extends Exception { + public ProtocolException(String msg, Throwable cause) { + super(msg, cause); + } + + public ProtocolException(String msg) { + super(msg); + } +} diff --git a/pkl-core/src/main/java/org/pkl/core/messaging/package-info.java b/pkl-core/src/main/java/org/pkl/core/messaging/package-info.java new file mode 100644 index 000000000..4253de960 --- /dev/null +++ b/pkl-core/src/main/java/org/pkl/core/messaging/package-info.java @@ -0,0 +1,4 @@ +@NonnullByDefault +package org.pkl.core.messaging; + +import org.pkl.core.util.NonnullByDefault; diff --git a/pkl-core/src/main/java/org/pkl/core/module/ExternalModuleResolver.java b/pkl-core/src/main/java/org/pkl/core/module/ExternalModuleResolver.java new file mode 100644 index 000000000..5d2c854e6 --- /dev/null +++ b/pkl-core/src/main/java/org/pkl/core/module/ExternalModuleResolver.java @@ -0,0 +1,129 @@ +/* + * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.core.module; + +import java.io.IOException; +import java.net.URI; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Random; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Future; +import org.pkl.core.SecurityManager; +import org.pkl.core.SecurityManagerException; +import org.pkl.core.messaging.MessageTransport; +import org.pkl.core.messaging.MessageTransports; +import org.pkl.core.messaging.Messages.ListModulesRequest; +import org.pkl.core.messaging.Messages.ListModulesResponse; +import org.pkl.core.messaging.Messages.ReadModuleRequest; +import org.pkl.core.messaging.Messages.ReadModuleResponse; +import org.pkl.core.messaging.ProtocolException; + +public class ExternalModuleResolver { + private final MessageTransport transport; + private final long evaluatorId; + private final Map> readResponses = new ConcurrentHashMap<>(); + private final Map>> listResponses = new ConcurrentHashMap<>(); + + public ExternalModuleResolver(MessageTransport transport, long evaluatorId) { + this.transport = transport; + this.evaluatorId = evaluatorId; + } + + public List listElements(SecurityManager securityManager, URI uri) + throws IOException, SecurityManagerException { + securityManager.checkResolveModule(uri); + return doListElements(uri); + } + + public boolean hasElement(SecurityManager securityManager, URI uri) + throws SecurityManagerException { + securityManager.checkResolveModule(uri); + try { + doReadModule(uri); + return true; + } catch (IOException e) { + return false; + } + } + + public String resolveModule(SecurityManager securityManager, URI uri) + throws IOException, SecurityManagerException { + securityManager.checkResolveModule(uri); + return doReadModule(uri); + } + + private String doReadModule(URI moduleUri) throws IOException { + return MessageTransports.resolveFuture( + readResponses.computeIfAbsent( + moduleUri, + (uri) -> { + var future = new CompletableFuture(); + var request = new ReadModuleRequest(new Random().nextLong(), evaluatorId, uri); + try { + transport.send( + request, + (response) -> { + if (response instanceof ReadModuleResponse resp) { + if (resp.error() != null) { + future.completeExceptionally(new IOException(resp.error())); + } else if (resp.contents() != null) { + future.complete(resp.contents()); + } else { + future.complete(""); + } + } else { + future.completeExceptionally(new ProtocolException("unexpected response")); + } + }); + } catch (ProtocolException | IOException e) { + future.completeExceptionally(e); + } + return future; + })); + } + + private List doListElements(URI baseUri) throws IOException { + return MessageTransports.resolveFuture( + listResponses.computeIfAbsent( + baseUri, + (uri) -> { + var future = new CompletableFuture>(); + var request = new ListModulesRequest(new Random().nextLong(), evaluatorId, uri); + try { + transport.send( + request, + (response) -> { + if (response instanceof ListModulesResponse resp) { + if (resp.error() != null) { + future.completeExceptionally(new IOException(resp.error())); + } else { + future.complete( + Objects.requireNonNullElseGet(resp.pathElements(), List::of)); + } + } else { + future.completeExceptionally(new ProtocolException("unexpected response")); + } + }); + } catch (ProtocolException | IOException e) { + future.completeExceptionally(e); + } + return future; + })); + } +} diff --git a/pkl-core/src/main/java/org/pkl/core/module/ModuleKeyFactories.java b/pkl-core/src/main/java/org/pkl/core/module/ModuleKeyFactories.java index e71c73859..e242377f6 100644 --- a/pkl-core/src/main/java/org/pkl/core/module/ModuleKeyFactories.java +++ b/pkl-core/src/main/java/org/pkl/core/module/ModuleKeyFactories.java @@ -15,6 +15,7 @@ */ package org.pkl.core.module; +import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.nio.file.FileSystemNotFoundException; @@ -25,6 +26,10 @@ import java.util.List; import java.util.Optional; import java.util.ServiceLoader; +import javax.annotation.concurrent.GuardedBy; +import org.pkl.core.externalreader.ExternalReaderProcess; +import org.pkl.core.externalreader.ExternalReaderProcessException; +import org.pkl.core.util.ErrorMessages; import org.pkl.core.util.IoUtils; /** Utilities for obtaining and using module key factories. */ @@ -72,7 +77,27 @@ public static ModuleKeyFactory classPath(ClassLoader classLoader) { return new ClassPath(classLoader); } - /** Closes the given factories, ignoring any exceptions. */ + /** + * Returns a factory for external reader module keys + * + *

NOTE: {@code process} needs to be {@link ExternalReaderProcess#close closed} to avoid + * resource leaks. + */ + public static ModuleKeyFactory externalProcess(String scheme, ExternalReaderProcess process) { + return new ExternalProcess(scheme, process, 0); + } + + public static ModuleKeyFactory externalProcess( + String scheme, ExternalReaderProcess process, long evaluatorId) { + return new ExternalProcess(scheme, process, evaluatorId); + } + + /** + * Closes the given factories, ignoring any exceptions. + * + * @deprecated Replaced by {@link org.pkl.core.util.Readers#closeQuietly}. + */ + @Deprecated(since = "0.27.0", forRemoval = true) public static void closeQuietly(Iterable factories) { for (ModuleKeyFactory factory : factories) { try { @@ -225,4 +250,48 @@ private static class FromServiceProviders { INSTANCE = Collections.unmodifiableList(factories); } } + + /** Represents a module from an external reader process. */ + private static final class ExternalProcess implements ModuleKeyFactory { + private final String scheme; + private final ExternalReaderProcess process; + private final long evaluatorId; + + @GuardedBy("this") + private ExternalModuleResolver resolver; + + public ExternalProcess(String scheme, ExternalReaderProcess process, long evaluatorId) { + this.scheme = scheme; + this.process = process; + this.evaluatorId = evaluatorId; + } + + private synchronized ExternalModuleResolver getResolver() + throws ExternalReaderProcessException { + if (resolver != null) { + return resolver; + } + + resolver = new ExternalModuleResolver(process.getTransport(), evaluatorId); + return resolver; + } + + public Optional create(URI uri) + throws URISyntaxException, ExternalReaderProcessException, IOException { + if (!scheme.equalsIgnoreCase(uri.getScheme())) return Optional.empty(); + + var spec = process.getModuleReaderSpec(scheme); + if (spec == null) { + throw new ExternalReaderProcessException( + ErrorMessages.create("externalReaderDoesNotSupportScheme", "module", scheme)); + } + + return Optional.of(ModuleKeys.externalResolver(uri, spec, getResolver())); + } + + @Override + public void close() { + process.close(); + } + } } diff --git a/pkl-core/src/main/java/org/pkl/core/module/ModuleKeyFactory.java b/pkl-core/src/main/java/org/pkl/core/module/ModuleKeyFactory.java index 33552051a..7bc17cc98 100644 --- a/pkl-core/src/main/java/org/pkl/core/module/ModuleKeyFactory.java +++ b/pkl-core/src/main/java/org/pkl/core/module/ModuleKeyFactory.java @@ -15,9 +15,11 @@ */ package org.pkl.core.module; +import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.util.Optional; +import org.pkl.core.externalreader.ExternalReaderProcessException; /** A factory for {@link ModuleKey}s. */ public interface ModuleKeyFactory extends AutoCloseable { @@ -35,7 +37,8 @@ public interface ModuleKeyFactory extends AutoCloseable { * @param uri an absolute normalized URI * @return a module key for the given URI */ - Optional create(URI uri) throws URISyntaxException; + Optional create(URI uri) + throws URISyntaxException, ExternalReaderProcessException, IOException; /** * Closes this factory, releasing any resources held. See the documentation of factory methods in diff --git a/pkl-core/src/main/java/org/pkl/core/module/ModuleKeys.java b/pkl-core/src/main/java/org/pkl/core/module/ModuleKeys.java index b633a663d..e288e3dbb 100644 --- a/pkl-core/src/main/java/org/pkl/core/module/ModuleKeys.java +++ b/pkl-core/src/main/java/org/pkl/core/module/ModuleKeys.java @@ -29,6 +29,8 @@ import java.util.Map; import org.pkl.core.SecurityManager; import org.pkl.core.SecurityManagerException; +import org.pkl.core.externalreader.ExternalReaderProcessException; +import org.pkl.core.messaging.Messages.*; import org.pkl.core.packages.Dependency; import org.pkl.core.packages.Dependency.LocalDependency; import org.pkl.core.packages.PackageAssetUri; @@ -127,6 +129,12 @@ public static ModuleKey projectpackage(URI uri) throws URISyntaxException { return new ProjectPackage(assetUri); } + /** Creates a module key for an externally read module. */ + public static ModuleKey externalResolver( + URI uri, ModuleReaderSpec spec, ExternalModuleResolver resolver) throws URISyntaxException { + return new ExternalResolver(uri, spec, resolver); + } + /** * Creates a module key that behaves like {@code delegate}, except that it returns {@code text} as * its loaded source. @@ -165,7 +173,7 @@ public ResolvedModuleKey resolve(SecurityManager securityManager) { } @Override - public boolean hasHierarchicalUris() { + public boolean hasHierarchicalUris() throws IOException, ExternalReaderProcessException { return delegate.hasHierarchicalUris(); } @@ -175,19 +183,19 @@ public boolean isLocal() { } @Override - public boolean isGlobbable() { + public boolean isGlobbable() throws IOException, ExternalReaderProcessException { return delegate.isGlobbable(); } @Override public boolean hasElement(SecurityManager securityManager, URI uri) - throws IOException, SecurityManagerException { + throws IOException, SecurityManagerException, ExternalReaderProcessException { return delegate.hasElement(securityManager, uri); } @Override public List listElements(SecurityManager securityManager, URI baseUri) - throws IOException, SecurityManagerException { + throws IOException, SecurityManagerException, ExternalReaderProcessException { return delegate.listElements(securityManager, baseUri); } } @@ -397,7 +405,6 @@ public ResolvedModuleKey resolve(SecurityManager securityManager) } private static final class ClassPath implements ModuleKey { - final URI uri; final ClassLoader classLoader; @@ -460,7 +467,6 @@ private String getResourcePath() { } private static class Http implements ModuleKey { - private final URI uri; Http(URI uri) { @@ -550,7 +556,6 @@ public ResolvedModuleKey resolve(SecurityManager securityManager) } private abstract static class AbstractPackage implements ModuleKey { - protected final PackageAssetUri packageAssetUri; AbstractPackage(PackageAssetUri packageAssetUri) { @@ -663,6 +668,7 @@ public boolean hasElement(SecurityManager securityManager, URI elementUri) * an internal implementation detail, and we do not expect a module to declare this. */ public static class ProjectPackage extends AbstractPackage { + ProjectPackage(PackageAssetUri packageAssetUri) { super(packageAssetUri); } @@ -712,7 +718,7 @@ public ResolvedModuleKey resolve(SecurityManager securityManager) @Override public List listElements(SecurityManager securityManager, URI baseUri) - throws IOException, SecurityManagerException { + throws IOException, SecurityManagerException, ExternalReaderProcessException { securityManager.checkResolveModule(baseUri); var packageAssetUri = PackageAssetUri.create(baseUri); var dependency = @@ -733,7 +739,7 @@ public List listElements(SecurityManager securityManager, URI baseU @Override public boolean hasElement(SecurityManager securityManager, URI elementUri) - throws IOException, SecurityManagerException { + throws IOException, SecurityManagerException, ExternalReaderProcessException { securityManager.checkResolveModule(elementUri); var packageAssetUri = PackageAssetUri.create(elementUri); var dependency = @@ -769,4 +775,56 @@ public boolean hasElement(SecurityManager securityManager, URI elementUri) return projectResolver.getResolvedDependenciesForPackage(packageUri, dependencyMetadata); } } + + public static class ExternalResolver implements ModuleKey { + + private final URI uri; + private final ModuleReaderSpec spec; + private final ExternalModuleResolver resolver; + + public ExternalResolver(URI uri, ModuleReaderSpec spec, ExternalModuleResolver resolver) { + this.uri = uri; + this.spec = spec; + this.resolver = resolver; + } + + @Override + public boolean isLocal() { + return spec.isLocal(); + } + + @Override + public boolean hasHierarchicalUris() { + return spec.hasHierarchicalUris(); + } + + @Override + public boolean isGlobbable() { + return spec.isGlobbable(); + } + + @Override + public URI getUri() { + return uri; + } + + @Override + public List listElements(SecurityManager securityManager, URI baseUri) + throws IOException, SecurityManagerException { + return resolver.listElements(securityManager, baseUri); + } + + @Override + public ResolvedModuleKey resolve(SecurityManager securityManager) + throws IOException, SecurityManagerException { + var contents = resolver.resolveModule(securityManager, uri); + return ResolvedModuleKeys.virtual(this, uri, contents, true); + } + + @Override + public boolean hasElement(SecurityManager securityManager, URI elementUri) + throws IOException, SecurityManagerException { + return resolver.hasElement(securityManager, elementUri); + } + } } diff --git a/pkl-core/src/main/java/org/pkl/core/packages/Checksums.java b/pkl-core/src/main/java/org/pkl/core/packages/Checksums.java index 232c24d57..9b173f0e2 100644 --- a/pkl-core/src/main/java/org/pkl/core/packages/Checksums.java +++ b/pkl-core/src/main/java/org/pkl/core/packages/Checksums.java @@ -16,6 +16,7 @@ package org.pkl.core.packages; import java.util.Objects; +import org.pkl.core.util.Nullable; public final class Checksums { private final String sha256; @@ -34,7 +35,7 @@ public String getSha256() { } @Override - public boolean equals(Object o) { + public boolean equals(@Nullable Object o) { if (this == o) { return true; } diff --git a/pkl-core/src/main/java/org/pkl/core/project/Project.java b/pkl-core/src/main/java/org/pkl/core/project/Project.java index 0d15ec335..59d122cf0 100644 --- a/pkl-core/src/main/java/org/pkl/core/project/Project.java +++ b/pkl-core/src/main/java/org/pkl/core/project/Project.java @@ -522,6 +522,8 @@ public EvaluatorSettings( modulePath, timeout, rootDir, + null, + null, null); } diff --git a/pkl-core/src/main/java/org/pkl/core/resource/ExternalResourceResolver.java b/pkl-core/src/main/java/org/pkl/core/resource/ExternalResourceResolver.java new file mode 100644 index 000000000..bde9a7571 --- /dev/null +++ b/pkl-core/src/main/java/org/pkl/core/resource/ExternalResourceResolver.java @@ -0,0 +1,127 @@ +/* + * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.core.resource; + +import java.io.IOException; +import java.net.URI; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Random; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Future; +import org.pkl.core.SecurityManager; +import org.pkl.core.SecurityManagerException; +import org.pkl.core.messaging.MessageTransport; +import org.pkl.core.messaging.MessageTransports; +import org.pkl.core.messaging.Messages.*; +import org.pkl.core.messaging.ProtocolException; +import org.pkl.core.module.PathElement; + +public class ExternalResourceResolver { + private final MessageTransport transport; + private final long evaluatorId; + private final Map> readResponses = new ConcurrentHashMap<>(); + private final Map>> listResponses = new ConcurrentHashMap<>(); + + public ExternalResourceResolver(MessageTransport transport, long evaluatorId) { + this.transport = transport; + this.evaluatorId = evaluatorId; + } + + public Optional read(URI uri) throws IOException { + var result = doRead(uri); + return Optional.of(new Resource(uri, result)); + } + + public boolean hasElement(org.pkl.core.SecurityManager securityManager, URI elementUri) + throws SecurityManagerException { + securityManager.checkResolveResource(elementUri); + try { + doRead(elementUri); + return true; + } catch (IOException e) { + return false; + } + } + + public List listElements(SecurityManager securityManager, URI baseUri) + throws IOException, SecurityManagerException { + securityManager.checkResolveResource(baseUri); + return doListElements(baseUri); + } + + public List doListElements(URI baseUri) throws IOException { + return MessageTransports.resolveFuture( + listResponses.computeIfAbsent( + baseUri, + (uri) -> { + var future = new CompletableFuture>(); + var request = new ListResourcesRequest(new Random().nextLong(), evaluatorId, uri); + try { + transport.send( + request, + (response) -> { + if (response instanceof ListResourcesResponse resp) { + if (resp.error() != null) { + future.completeExceptionally(new IOException(resp.error())); + } else { + future.complete( + Objects.requireNonNullElseGet(resp.pathElements(), List::of)); + } + } else { + future.completeExceptionally(new ProtocolException("unexpected response")); + } + }); + } catch (ProtocolException | IOException e) { + future.completeExceptionally(e); + } + return future; + })); + } + + public byte[] doRead(URI baseUri) throws IOException { + return MessageTransports.resolveFuture( + readResponses.computeIfAbsent( + baseUri, + (uri) -> { + var future = new CompletableFuture(); + var request = new ReadResourceRequest(new Random().nextLong(), evaluatorId, uri); + try { + transport.send( + request, + (response) -> { + if (response instanceof ReadResourceResponse resp) { + if (resp.error() != null) { + future.completeExceptionally(new IOException(resp.error())); + } else if (resp.contents() != null) { + future.complete(resp.contents()); + } else { + future.complete(new byte[0]); + } + } else { + future.completeExceptionally(new ProtocolException("unexpected response")); + } + }); + } catch (ProtocolException | IOException e) { + future.completeExceptionally(e); + } + return future; + })); + } +} diff --git a/pkl-core/src/main/java/org/pkl/core/resource/ResourceReader.java b/pkl-core/src/main/java/org/pkl/core/resource/ResourceReader.java index dc8c0bc3e..40e3f39eb 100644 --- a/pkl-core/src/main/java/org/pkl/core/resource/ResourceReader.java +++ b/pkl-core/src/main/java/org/pkl/core/resource/ResourceReader.java @@ -20,6 +20,7 @@ import java.net.URISyntaxException; import java.util.Optional; import org.pkl.core.SecurityManagerException; +import org.pkl.core.externalreader.ExternalReaderProcessException; import org.pkl.core.runtime.ReaderBase; /** @@ -29,7 +30,7 @@ * *

See {@link ResourceReaders} for predefined resource readers. */ -public interface ResourceReader extends ReaderBase { +public interface ResourceReader extends ReaderBase, AutoCloseable { /** The URI scheme associated with resources read by this resource reader. */ String getUriScheme(); @@ -54,5 +55,16 @@ public interface ResourceReader extends ReaderBase { * manager. * */ - Optional read(URI uri) throws IOException, URISyntaxException, SecurityManagerException; + Optional read(URI uri) + throws IOException, + URISyntaxException, + SecurityManagerException, + ExternalReaderProcessException; + + /** + * Closes this reader, releasing any resources held. See the documentation of factory methods in + * {@link ResourceReaders} for which factories need to be closed. + */ + @Override + default void close() {} } diff --git a/pkl-core/src/main/java/org/pkl/core/resource/ResourceReaders.java b/pkl-core/src/main/java/org/pkl/core/resource/ResourceReaders.java index 7f08702fc..008f667f4 100644 --- a/pkl-core/src/main/java/org/pkl/core/resource/ResourceReaders.java +++ b/pkl-core/src/main/java/org/pkl/core/resource/ResourceReaders.java @@ -29,6 +29,9 @@ import java.util.ServiceLoader; import org.pkl.core.SecurityManager; import org.pkl.core.SecurityManagerException; +import org.pkl.core.externalreader.ExternalReaderProcess; +import org.pkl.core.externalreader.ExternalReaderProcessException; +import org.pkl.core.messaging.Messages.*; import org.pkl.core.module.FileResolver; import org.pkl.core.module.ModulePathResolver; import org.pkl.core.module.PathElement; @@ -137,6 +140,21 @@ public static List fromServiceProviders() { return FromServiceProviders.INSTANCE; } + public static ResourceReader externalProcess( + String scheme, ExternalReaderProcess externalReaderProcess) { + return new ExternalProcess(scheme, externalReaderProcess, 0); + } + + public static ResourceReader externalProcess( + String scheme, ExternalReaderProcess externalReaderProcess, long evaluatorId) { + return new ExternalProcess(scheme, externalReaderProcess, evaluatorId); + } + + public static ResourceReader externalResolver( + ResourceReaderSpec spec, ExternalResourceResolver resolver) { + return new ExternalResolver(spec, resolver); + } + private static final class EnvironmentVariable implements ResourceReader { static final ResourceReader INSTANCE = new EnvironmentVariable(); @@ -521,7 +539,7 @@ public boolean hasFragmentPaths() { @Override public List listElements(SecurityManager securityManager, URI baseUri) - throws IOException, SecurityManagerException { + throws IOException, SecurityManagerException, ExternalReaderProcessException { securityManager.checkResolveResource(baseUri); var packageAssetUri = PackageAssetUri.create(baseUri); var dependency = @@ -543,7 +561,7 @@ public List listElements(SecurityManager securityManager, URI baseU @Override public boolean hasElement(SecurityManager securityManager, URI elementUri) - throws IOException, SecurityManagerException { + throws IOException, SecurityManagerException, ExternalReaderProcessException { securityManager.checkResolveResource(elementUri); var packageAssetUri = PackageAssetUri.create(elementUri); var dependency = @@ -585,6 +603,7 @@ private ProjectDependenciesManager getProjectDepsResolver() { } private static class FromServiceProviders { + private static final List INSTANCE; static { @@ -594,4 +613,113 @@ private static class FromServiceProviders { INSTANCE = Collections.unmodifiableList(readers); } } + + private static final class ExternalProcess implements ResourceReader { + private final String scheme; + private final ExternalReaderProcess process; + private final long evaluatorId; + private ExternalResolver underlying; + + public ExternalProcess(String scheme, ExternalReaderProcess process, long evaluatorId) { + this.scheme = scheme; + this.process = process; + this.evaluatorId = evaluatorId; + } + + private ExternalResolver getUnderlyingReader() + throws ExternalReaderProcessException, IOException { + if (underlying != null) { + return underlying; + } + + var spec = process.getResourceReaderSpec(scheme); + if (spec == null) { + throw new ExternalReaderProcessException( + ErrorMessages.create("externalReaderDoesNotSupportScheme", "resource", scheme)); + } + underlying = + new ExternalResolver( + spec, new ExternalResourceResolver(process.getTransport(), evaluatorId)); + return underlying; + } + + @Override + public String getUriScheme() { + return scheme; + } + + @Override + public boolean hasHierarchicalUris() throws ExternalReaderProcessException, IOException { + return getUnderlyingReader().hasHierarchicalUris(); + } + + @Override + public boolean isGlobbable() throws ExternalReaderProcessException, IOException { + return getUnderlyingReader().isGlobbable(); + } + + @Override + public Optional read(URI uri) throws IOException, ExternalReaderProcessException { + return getUnderlyingReader().read(uri); + } + + @Override + public boolean hasElement(SecurityManager securityManager, URI elementUri) + throws IOException, SecurityManagerException, ExternalReaderProcessException { + return getUnderlyingReader().hasElement(securityManager, elementUri); + } + + @Override + public List listElements(SecurityManager securityManager, URI baseUri) + throws IOException, SecurityManagerException, ExternalReaderProcessException { + return getUnderlyingReader().listElements(securityManager, baseUri); + } + + @Override + public void close() { + process.close(); + } + } + + private static final class ExternalResolver implements ResourceReader { + private final ResourceReaderSpec readerSpec; + private final ExternalResourceResolver resolver; + + public ExternalResolver(ResourceReaderSpec readerSpec, ExternalResourceResolver resolver) { + this.readerSpec = readerSpec; + this.resolver = resolver; + } + + @Override + public boolean hasHierarchicalUris() { + return readerSpec.hasHierarchicalUris(); + } + + @Override + public boolean isGlobbable() { + return readerSpec.isGlobbable(); + } + + @Override + public String getUriScheme() { + return readerSpec.scheme(); + } + + @Override + public Optional read(URI uri) throws IOException { + return resolver.read(uri); + } + + @Override + public boolean hasElement(org.pkl.core.SecurityManager securityManager, URI elementUri) + throws SecurityManagerException { + return resolver.hasElement(securityManager, elementUri); + } + + @Override + public List listElements(SecurityManager securityManager, URI baseUri) + throws IOException, SecurityManagerException { + return resolver.listElements(securityManager, baseUri); + } + } } diff --git a/pkl-core/src/main/java/org/pkl/core/runtime/ModuleResolver.java b/pkl-core/src/main/java/org/pkl/core/runtime/ModuleResolver.java index 3fe7bc62f..e90719f18 100644 --- a/pkl-core/src/main/java/org/pkl/core/runtime/ModuleResolver.java +++ b/pkl-core/src/main/java/org/pkl/core/runtime/ModuleResolver.java @@ -16,11 +16,13 @@ package org.pkl.core.runtime; import com.oracle.truffle.api.nodes.Node; +import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.util.Collection; import java.util.Optional; import org.pkl.core.ModuleSource; +import org.pkl.core.externalreader.ExternalReaderProcessException; import org.pkl.core.module.ModuleKey; import org.pkl.core.module.ModuleKeyFactory; import org.pkl.core.module.ModuleKeys; @@ -82,6 +84,18 @@ public ModuleKey resolve(URI moduleUri, @Nullable Node importNode) { .evalError("invalidModuleUri", moduleUri) .withHint(e.getReason()) .build(); + } catch (ExternalReaderProcessException e) { + throw new VmExceptionBuilder() + .withOptionalLocation(importNode) + .evalError("externalReaderFailure") + .withCause(e) + .build(); + } catch (IOException e) { + throw new VmExceptionBuilder() + .withOptionalLocation(importNode) + .evalError("ioErrorLoadingModule") + .withCause(e) + .build(); } if (key.isPresent()) return key.get(); } diff --git a/pkl-core/src/main/java/org/pkl/core/runtime/ReaderBase.java b/pkl-core/src/main/java/org/pkl/core/runtime/ReaderBase.java index 1fb5909bd..e1f461d25 100644 --- a/pkl-core/src/main/java/org/pkl/core/runtime/ReaderBase.java +++ b/pkl-core/src/main/java/org/pkl/core/runtime/ReaderBase.java @@ -20,6 +20,7 @@ import java.util.List; import org.pkl.core.SecurityManager; import org.pkl.core.SecurityManagerException; +import org.pkl.core.externalreader.ExternalReaderProcessException; import org.pkl.core.module.ModuleKey; import org.pkl.core.module.PathElement; import org.pkl.core.util.IoUtils; @@ -29,10 +30,10 @@ public interface ReaderBase { * Tells if the URIs represented by this module key or resource reader should be interpreted as hierarchical. */ - boolean hasHierarchicalUris(); + boolean hasHierarchicalUris() throws ExternalReaderProcessException, IOException; /** Tells if this module key or resource reader supports globbing. */ - boolean isGlobbable(); + boolean isGlobbable() throws ExternalReaderProcessException, IOException; /** * Tells if relative paths of this URI should be resolved from {@link URI#getFragment()}, rather @@ -49,7 +50,7 @@ default boolean hasFragmentPaths() { * if either {@link #isGlobbable()} or {@link ModuleKey#isLocal()} returns true. */ default boolean hasElement(SecurityManager securityManager, URI elementUri) - throws IOException, SecurityManagerException { + throws IOException, SecurityManagerException, ExternalReaderProcessException { throw new UnsupportedOperationException(); } @@ -66,7 +67,7 @@ default boolean hasElement(SecurityManager securityManager, URI elementUri) * this reader. */ default List listElements(SecurityManager securityManager, URI baseUri) - throws IOException, SecurityManagerException { + throws IOException, SecurityManagerException, ExternalReaderProcessException { throw new UnsupportedOperationException(); } diff --git a/pkl-core/src/main/java/org/pkl/core/runtime/ResourceManager.java b/pkl-core/src/main/java/org/pkl/core/runtime/ResourceManager.java index 1992a6a26..17f2cbdf4 100644 --- a/pkl-core/src/main/java/org/pkl/core/runtime/ResourceManager.java +++ b/pkl-core/src/main/java/org/pkl/core/runtime/ResourceManager.java @@ -26,6 +26,7 @@ import java.util.Optional; import org.pkl.core.SecurityManager; import org.pkl.core.SecurityManagerException; +import org.pkl.core.externalreader.ExternalReaderProcessException; import org.pkl.core.http.HttpClientInitException; import org.pkl.core.packages.PackageLoadError; import org.pkl.core.resource.Resource; @@ -83,7 +84,10 @@ public Optional doRead(ResourceReader reader, URI uri, @Nullable Node re .withHint(e.getReason()) .withOptionalLocation(readNode) .build(); - } catch (SecurityManagerException | PackageLoadError | HttpClientInitException e) { + } catch (SecurityManagerException + | PackageLoadError + | HttpClientInitException + | ExternalReaderProcessException e) { throw new VmExceptionBuilder().withCause(e).withOptionalLocation(readNode).build(); } return resource; diff --git a/pkl-core/src/main/java/org/pkl/core/runtime/VmImportAnalyzer.java b/pkl-core/src/main/java/org/pkl/core/runtime/VmImportAnalyzer.java index c096e0b48..b883ba9f7 100644 --- a/pkl-core/src/main/java/org/pkl/core/runtime/VmImportAnalyzer.java +++ b/pkl-core/src/main/java/org/pkl/core/runtime/VmImportAnalyzer.java @@ -30,6 +30,7 @@ import org.pkl.core.SecurityManagerException; import org.pkl.core.ast.builder.ImportsAndReadsParser; import org.pkl.core.ast.builder.ImportsAndReadsParser.Entry; +import org.pkl.core.externalreader.ExternalReaderProcessException; import org.pkl.core.util.GlobResolver; import org.pkl.core.util.GlobResolver.InvalidGlobPatternException; import org.pkl.core.util.GlobResolver.ResolvedGlobElement; @@ -38,7 +39,10 @@ public class VmImportAnalyzer { @TruffleBoundary public static ImportGraph analyze(URI[] moduleUris, VmContext context) - throws IOException, URISyntaxException, SecurityManagerException { + throws IOException, + URISyntaxException, + SecurityManagerException, + ExternalReaderProcessException { var imports = new TreeMap>(); var resolvedImports = new TreeMap(); for (var moduleUri : moduleUris) { @@ -53,7 +57,10 @@ private static void analyzeSingle( VmContext context, Map> imports, Map resolvedImports) - throws IOException, URISyntaxException, SecurityManagerException { + throws IOException, + URISyntaxException, + SecurityManagerException, + ExternalReaderProcessException { var moduleResolver = context.getModuleResolver(); var securityManager = context.getSecurityManager(); var importsInModule = collectImports(moduleUri, moduleResolver, securityManager); @@ -71,7 +78,10 @@ private static void analyzeSingle( private static Set collectImports( URI moduleUri, ModuleResolver moduleResolver, SecurityManager securityManager) - throws IOException, URISyntaxException, SecurityManagerException { + throws IOException, + URISyntaxException, + SecurityManagerException, + ExternalReaderProcessException { var moduleKey = moduleResolver.resolve(moduleUri); var resolvedModuleKey = moduleKey.resolve(securityManager); List importsAndReads; diff --git a/pkl-core/src/main/java/org/pkl/core/service/ExecutorSpiImpl.java b/pkl-core/src/main/java/org/pkl/core/service/ExecutorSpiImpl.java index 0f2a86d31..a4fd373c3 100644 --- a/pkl-core/src/main/java/org/pkl/core/service/ExecutorSpiImpl.java +++ b/pkl-core/src/main/java/org/pkl/core/service/ExecutorSpiImpl.java @@ -33,6 +33,7 @@ import org.pkl.core.module.ModulePathResolver; import org.pkl.core.project.Project; import org.pkl.core.resource.ResourceReaders; +import org.pkl.core.util.Readers; import org.pkl.executor.spi.v1.ExecutorSpi; import org.pkl.executor.spi.v1.ExecutorSpiException; import org.pkl.executor.spi.v1.ExecutorSpiOptions; @@ -125,7 +126,8 @@ public String evaluatePath(Path modulePath, ExecutorSpiOptions options) { } catch (PklException e) { throw new ExecutorSpiException(e.getMessage(), e.getCause()); } finally { - ModuleKeyFactories.closeQuietly(builder.getModuleKeyFactories()); + Readers.closeQuietly(builder.getModuleKeyFactories()); + Readers.closeQuietly(builder.getResourceReaders()); } } diff --git a/pkl-core/src/main/java/org/pkl/core/stdlib/analyze/AnalyzeNodes.java b/pkl-core/src/main/java/org/pkl/core/stdlib/analyze/AnalyzeNodes.java index feb60998e..fdfaad1cc 100644 --- a/pkl-core/src/main/java/org/pkl/core/stdlib/analyze/AnalyzeNodes.java +++ b/pkl-core/src/main/java/org/pkl/core/stdlib/analyze/AnalyzeNodes.java @@ -23,6 +23,7 @@ import org.pkl.core.ImportGraph; import org.pkl.core.ImportGraph.Import; import org.pkl.core.SecurityManagerException; +import org.pkl.core.externalreader.ExternalReaderProcessException; import org.pkl.core.packages.PackageLoadError; import org.pkl.core.runtime.AnalyzeModule; import org.pkl.core.runtime.VmContext; @@ -91,7 +92,11 @@ protected Object eval(@SuppressWarnings("unused") VmTyped self, VmSet moduleUris try { var results = VmImportAnalyzer.analyze(uris, context); return importGraphFactory.create(results); - } catch (IOException | URISyntaxException | SecurityManagerException | PackageLoadError e) { + } catch (IOException + | URISyntaxException + | SecurityManagerException + | PackageLoadError + | ExternalReaderProcessException e) { throw exceptionBuilder().withCause(e).build(); } } diff --git a/pkl-core/src/main/java/org/pkl/core/stdlib/benchmark/OutputBenchmarkNodes.java b/pkl-core/src/main/java/org/pkl/core/stdlib/benchmark/OutputBenchmarkNodes.java index 5863e5f6b..1bce83c1a 100644 --- a/pkl-core/src/main/java/org/pkl/core/stdlib/benchmark/OutputBenchmarkNodes.java +++ b/pkl-core/src/main/java/org/pkl/core/stdlib/benchmark/OutputBenchmarkNodes.java @@ -25,6 +25,7 @@ import java.util.List; import org.pkl.core.SecurityManager; import org.pkl.core.SecurityManagerException; +import org.pkl.core.externalreader.ExternalReaderProcessException; import org.pkl.core.module.ModuleKey; import org.pkl.core.module.PathElement; import org.pkl.core.module.ResolvedModuleKey; @@ -108,7 +109,7 @@ public boolean isCached() { } @Override - public boolean hasHierarchicalUris() { + public boolean hasHierarchicalUris() throws IOException, ExternalReaderProcessException { return delegate.hasHierarchicalUris(); } @@ -118,19 +119,19 @@ public boolean isLocal() { } @Override - public boolean isGlobbable() { + public boolean isGlobbable() throws IOException, ExternalReaderProcessException { return delegate.isGlobbable(); } @Override public boolean hasElement(SecurityManager securityManager, URI uri) - throws IOException, SecurityManagerException { + throws IOException, SecurityManagerException, ExternalReaderProcessException { return delegate.hasElement(securityManager, uri); } @Override public List listElements(SecurityManager securityManager, URI baseUri) - throws IOException, SecurityManagerException { + throws IOException, SecurityManagerException, ExternalReaderProcessException { return delegate.listElements(securityManager, baseUri); } } diff --git a/pkl-core/src/main/java/org/pkl/core/util/GlobResolver.java b/pkl-core/src/main/java/org/pkl/core/util/GlobResolver.java index 7df582f65..c14f2779e 100644 --- a/pkl-core/src/main/java/org/pkl/core/util/GlobResolver.java +++ b/pkl-core/src/main/java/org/pkl/core/util/GlobResolver.java @@ -31,6 +31,7 @@ import org.pkl.core.PklBugException; import org.pkl.core.SecurityManager; import org.pkl.core.SecurityManagerException; +import org.pkl.core.externalreader.ExternalReaderProcessException; import org.pkl.core.module.ModuleKey; import org.pkl.core.module.PathElement; import org.pkl.core.runtime.ReaderBase; @@ -260,7 +261,7 @@ private static void resolveOpaqueGlob( URI globUri, Pattern pattern, Map result) - throws IOException, SecurityManagerException { + throws IOException, SecurityManagerException, ExternalReaderProcessException { var elements = reader.listElements(securityManager, globUri); for (var elem : sorted(elements)) { URI resolvedUri; @@ -318,7 +319,10 @@ private static List expandHierarchicalGlobPart( boolean isGlobStar, boolean hasAbsoluteGlob, MutableLong listElementCallCount) - throws IOException, SecurityManagerException, InvalidGlobPatternException { + throws IOException, + SecurityManagerException, + InvalidGlobPatternException, + ExternalReaderProcessException { var result = new ArrayList(); doExpandHierarchicalGlobPart( securityManager, @@ -343,7 +347,10 @@ private static void doExpandHierarchicalGlobPart( boolean hasAbsoluteGlob, MutableLong listElementCallCount, List result) - throws IOException, SecurityManagerException, InvalidGlobPatternException { + throws IOException, + SecurityManagerException, + InvalidGlobPatternException, + ExternalReaderProcessException { if (listElementCallCount.getAndIncrement() > maxListElements()) { throw new InvalidGlobPatternException(ErrorMessages.create("invalidGlobTooComplex")); @@ -384,7 +391,10 @@ private static void resolveHierarchicalGlob( boolean hasAbsoluteGlob, Map result, MutableLong listElementCallCount) - throws IOException, SecurityManagerException, InvalidGlobPatternException { + throws IOException, + SecurityManagerException, + InvalidGlobPatternException, + ExternalReaderProcessException { var isLeaf = idx == globPatternParts.length - 1; var patternPart = globPatternParts[idx]; if (isRegularPathPart(patternPart)) { @@ -481,7 +491,10 @@ public static Map resolveGlob( ModuleKey enclosingModuleKey, URI enclosingUri, String globPattern) - throws IOException, SecurityManagerException, InvalidGlobPatternException { + throws IOException, + SecurityManagerException, + InvalidGlobPatternException, + ExternalReaderProcessException { var result = new LinkedHashMap(); var hasAbsoluteGlob = globPattern.matches("\\w+:.*"); diff --git a/pkl-core/src/main/java/org/pkl/core/util/IoUtils.java b/pkl-core/src/main/java/org/pkl/core/util/IoUtils.java index 071742370..f7a8bbd79 100644 --- a/pkl-core/src/main/java/org/pkl/core/util/IoUtils.java +++ b/pkl-core/src/main/java/org/pkl/core/util/IoUtils.java @@ -38,6 +38,7 @@ import org.pkl.core.Platform; import org.pkl.core.SecurityManager; import org.pkl.core.SecurityManagerException; +import org.pkl.core.externalreader.ExternalReaderProcessException; import org.pkl.core.module.ModuleKey; import org.pkl.core.packages.PackageLoadError; import org.pkl.core.runtime.ReaderBase; @@ -317,7 +318,7 @@ public static URI resolve(ReaderBase reader, URI baseUri, URI importUri) { private static URI resolveTripleDotImport( SecurityManager securityManager, ModuleKey moduleKey, String tripleDotPath) - throws IOException, SecurityManagerException { + throws IOException, SecurityManagerException, ExternalReaderProcessException { var moduleKeyUri = moduleKey.getUri(); if (!moduleKey.isLocal() || !moduleKey.hasHierarchicalUris()) { throw new VmExceptionBuilder() @@ -363,7 +364,8 @@ public static Pair parseDependencyNotation(String importPath) { return Pair.of(importPath.substring(1, idx), importPath.substring(idx)); } - private static URI resolveProjectDependency(ModuleKey moduleKey, String notation) { + private static URI resolveProjectDependency(ModuleKey moduleKey, String notation) + throws IOException, ExternalReaderProcessException { var parsed = parseDependencyNotation(notation); var name = parsed.getFirst(); var path = parsed.getSecond(); @@ -395,7 +397,10 @@ private static URI resolveProjectDependency(ModuleKey moduleKey, String notation * dependency notation () */ public static URI resolve(SecurityManager securityManager, ModuleKey moduleKey, URI importUri) - throws URISyntaxException, IOException, SecurityManagerException { + throws URISyntaxException, + IOException, + SecurityManagerException, + ExternalReaderProcessException { if (importUri.isAbsolute()) { return moduleKey.resolveUri(importUri); } diff --git a/pkl-core/src/main/java/org/pkl/core/util/Readers.java b/pkl-core/src/main/java/org/pkl/core/util/Readers.java new file mode 100644 index 000000000..5760b9787 --- /dev/null +++ b/pkl-core/src/main/java/org/pkl/core/util/Readers.java @@ -0,0 +1,28 @@ +/* + * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.core.util; + +public class Readers { + /** Closes the given readers, ignoring any exceptions. */ + public static void closeQuietly(Iterable readers) { + for (var reader : readers) { + try { + reader.close(); + } catch (Exception ignored) { + } + } + } +} diff --git a/pkl-core/src/main/resources/org/pkl/core/errorMessages.properties b/pkl-core/src/main/resources/org/pkl/core/errorMessages.properties index 8a02a926c..193045e2d 100644 --- a/pkl-core/src/main/resources/org/pkl/core/errorMessages.properties +++ b/pkl-core/src/main/resources/org/pkl/core/errorMessages.properties @@ -1081,3 +1081,33 @@ Malformed proxy URI (expecting `http://[:]`): `{0}`. cannotAnalyzeBecauseSyntaxError=\ Found a syntax error when parsing module `{0}`. + +malformedMessageHeaderLength=\ +Malformed message header (expected size 2, but got {0}). + +malformedMessageHeaderException=\ +Malformed message header. + +malformedMessageHeaderUnrecognizedCode=\ +Malformed message header (unrecognized code `{0}`). + +unhandledMessageCode=\ +Unhandled decoding message code `{0}`. + +unhandledMessageType=\ +Unhandled encoding message type `{0}`. + +malformedMessageBody=\ +Malformed message body for message with code `{0}`. + +missingMessageParameter=\ +Missing message parameter `{0}` + +unknownRequestId=\ +Received response {0} for unknown request ID `{1}`. + +externalReaderFailure=\ +Failed to communicate with external reader process. + +externalReaderDoesNotSupportScheme=\ +External {0} reader does not support scheme `{1}`. diff --git a/pkl-core/src/test/kotlin/org/pkl/core/EvaluatorBuilderTest.kt b/pkl-core/src/test/kotlin/org/pkl/core/EvaluatorBuilderTest.kt index 3ae9624bf..4a13f11eb 100644 --- a/pkl-core/src/test/kotlin/org/pkl/core/EvaluatorBuilderTest.kt +++ b/pkl-core/src/test/kotlin/org/pkl/core/EvaluatorBuilderTest.kt @@ -71,8 +71,10 @@ class EvaluatorBuilderTest { fun `sets evaluator settings from project`() { val projectPath = Path.of(javaClass.getResource("project/project1/PklProject")!!.toURI()) val project = Project.loadFromPath(projectPath, SecurityManagers.defaultManager, null) - val projectDir = Path.of(javaClass.getResource("project/project1/PklProject")!!.toURI()).parent - val builder = EvaluatorBuilder.unconfigured().applyFromProject(project) + val projectDir = projectPath.parent + val builder = EvaluatorBuilder.unconfigured() + val moduleKeyFactoryCount = builder.moduleKeyFactories.size + builder.applyFromProject(project) assertThat(builder.allowedResources.map { it.pattern() }).isEqualTo(listOf("foo:", "bar:")) assertThat(builder.allowedModules.map { it.pattern() }).isEqualTo(listOf("baz:", "biz:")) assertThat(builder.externalProperties).isEqualTo(mapOf("one" to "1")) @@ -80,5 +82,9 @@ class EvaluatorBuilderTest { assertThat(builder.moduleCacheDir).isEqualTo(projectDir.resolve("my-cache-dir/")) assertThat(builder.rootDir).isEqualTo(projectDir.resolve("my-root-dir/")) assertThat(builder.timeout).isEqualTo(Duration.ofMinutes(5L)) + assertThat(builder.moduleKeyFactories.size - moduleKeyFactoryCount) + .isEqualTo(3) // two external readers, one module path + assertThat(builder.resourceReaders.find { it.uriScheme == "scheme3" }).isNotNull + assertThat(builder.resourceReaders.find { it.uriScheme == "scheme4" }).isNotNull } } diff --git a/pkl-server/src/main/kotlin/org/pkl/server/MessageEncoders.kt b/pkl-core/src/test/kotlin/org/pkl/core/externalreader/ExternalModuleReader.kt similarity index 61% rename from pkl-server/src/main/kotlin/org/pkl/server/MessageEncoders.kt rename to pkl-core/src/test/kotlin/org/pkl/core/externalreader/ExternalModuleReader.kt index b902b56f8..e9be21d6f 100644 --- a/pkl-server/src/main/kotlin/org/pkl/server/MessageEncoders.kt +++ b/pkl-core/src/test/kotlin/org/pkl/core/externalreader/ExternalModuleReader.kt @@ -13,16 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.pkl.server +package org.pkl.core.externalreader -import java.io.OutputStream -import org.msgpack.core.MessagePack -import org.msgpack.core.MessagePacker +import java.net.URI +import org.pkl.core.messaging.Messages.ModuleReaderSpec -/** Factory methods for creating [MessageEncoder]s. */ -internal object MessageEncoders { - fun into(stream: OutputStream): MessageEncoder = - MessagePackEncoder(MessagePack.newDefaultPacker(stream)) +/** An external module reader, to be used with [ExternalReaderRuntime]. */ +interface ExternalModuleReader : ExternalReaderBase { + val isLocal: Boolean - fun into(packer: MessagePacker): MessageEncoder = MessagePackEncoder(packer) + fun read(uri: URI): String + + val spec: ModuleReaderSpec + get() = ModuleReaderSpec(scheme, hasHierarchicalUris, isLocal, isGlobbable) } diff --git a/pkl-core/src/test/kotlin/org/pkl/core/externalreader/ExternalProcessProcessReaderMessagePackCodecTest.kt b/pkl-core/src/test/kotlin/org/pkl/core/externalreader/ExternalProcessProcessReaderMessagePackCodecTest.kt new file mode 100644 index 000000000..235524fdc --- /dev/null +++ b/pkl-core/src/test/kotlin/org/pkl/core/externalreader/ExternalProcessProcessReaderMessagePackCodecTest.kt @@ -0,0 +1,73 @@ +/* + * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.core.externalreader + +import java.io.PipedInputStream +import java.io.PipedOutputStream +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import org.msgpack.core.MessagePack +import org.pkl.core.externalreader.ExternalReaderMessages.* +import org.pkl.core.messaging.* + +class ExternalProcessProcessReaderMessagePackCodecTest { + private val encoder: MessageEncoder + private val decoder: MessageDecoder + + init { + val inputStream = PipedInputStream() + val outputStream = PipedOutputStream(inputStream) + encoder = ExternalReaderMessagePackEncoder(MessagePack.newDefaultPacker(outputStream)) + decoder = ExternalReaderMessagePackDecoder(MessagePack.newDefaultUnpacker(inputStream)) + } + + private fun roundtrip(message: Message) { + encoder.encode(message) + val decoded = decoder.decode() + assertThat(decoded).isEqualTo(message) + } + + @Test + fun `round-trip InitializeModuleReaderRequest`() { + roundtrip(InitializeModuleReaderRequest(123, "my-scheme")) + } + + @Test + fun `round-trip InitializeResourceReaderRequest`() { + roundtrip(InitializeResourceReaderRequest(123, "my-scheme")) + } + + @Test + fun `round-trip InitializeModuleReaderResponse`() { + roundtrip(InitializeModuleReaderResponse(123, null)) + roundtrip( + InitializeModuleReaderResponse(123, Messages.ModuleReaderSpec("my-scheme", true, true, true)) + ) + } + + @Test + fun `round-trip InitializeResourceReaderResponse`() { + roundtrip(InitializeResourceReaderResponse(123, null)) + roundtrip( + InitializeResourceReaderResponse(123, Messages.ResourceReaderSpec("my-scheme", true, true)) + ) + } + + @Test + fun `round-trip CloseExternalProcess`() { + roundtrip(CloseExternalProcess()) + } +} diff --git a/pkl-server/src/main/kotlin/org/pkl/server/MessageTransport.kt b/pkl-core/src/test/kotlin/org/pkl/core/externalreader/ExternalReaderBase.kt similarity index 63% rename from pkl-server/src/main/kotlin/org/pkl/server/MessageTransport.kt rename to pkl-core/src/test/kotlin/org/pkl/core/externalreader/ExternalReaderBase.kt index a6bfee159..92d0338fe 100644 --- a/pkl-server/src/main/kotlin/org/pkl/server/MessageTransport.kt +++ b/pkl-core/src/test/kotlin/org/pkl/core/externalreader/ExternalReaderBase.kt @@ -13,15 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.pkl.server +package org.pkl.core.externalreader -/** A bidirectional transport for sending and receiving messages. */ -interface MessageTransport : AutoCloseable { - fun start(oneWayHandler: (OneWayMessage) -> Unit, requestHandler: (RequestMessage) -> Unit) +import java.net.URI +import org.pkl.core.module.PathElement - fun send(message: OneWayMessage) +/** Base interface for external module and resource readers. */ +interface ExternalReaderBase { + val scheme: String - fun send(message: RequestMessage, responseHandler: (ResponseMessage) -> Unit) + val hasHierarchicalUris: Boolean - fun send(message: ResponseMessage) + val isGlobbable: Boolean + + fun listElements(uri: URI): List } diff --git a/pkl-core/src/test/kotlin/org/pkl/core/externalreader/ExternalReaderRuntime.kt b/pkl-core/src/test/kotlin/org/pkl/core/externalreader/ExternalReaderRuntime.kt new file mode 100644 index 000000000..95b7dba32 --- /dev/null +++ b/pkl-core/src/test/kotlin/org/pkl/core/externalreader/ExternalReaderRuntime.kt @@ -0,0 +1,199 @@ +/* + * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.core.externalreader + +import java.io.IOException +import org.pkl.core.externalreader.ExternalReaderMessages.* +import org.pkl.core.messaging.Message +import org.pkl.core.messaging.MessageTransport +import org.pkl.core.messaging.Messages.* +import org.pkl.core.messaging.ProtocolException +import org.pkl.core.util.Nullable + +/** An implementation of the client side of the external reader flow */ +class ExternalReaderRuntime( + private val moduleReaders: List, + private val resourceReaders: List, + private val transport: MessageTransport +) { + /** Close the runtime and its transport. */ + fun close() { + transport.close() + } + + private fun findModuleReader(scheme: String): @Nullable ExternalModuleReader? { + for (moduleReader in moduleReaders) { + if (moduleReader.scheme.equals(scheme, ignoreCase = true)) { + return moduleReader + } + } + return null + } + + private fun findResourceReader(scheme: String): @Nullable ExternalResourceReader? { + for (resourceReader in resourceReaders) { + if (resourceReader.scheme.equals(scheme, ignoreCase = true)) { + return resourceReader + } + } + return null + } + + /** + * Start the runtime so it can respond to incoming messages on its transport. + * + * Blocks until the underlying transport is closed. + */ + @Throws(ProtocolException::class, IOException::class) + fun run() { + transport.start( + { msg: Message.OneWay -> + if (msg.type() == Message.Type.CLOSE_EXTERNAL_PROCESS) { + close() + } else { + throw ProtocolException("Unexpected incoming one-way message: $msg") + } + }, + { msg: Message.Request -> + when (msg.type()) { + Message.Type.INITIALIZE_MODULE_READER_REQUEST -> { + val req = msg as InitializeModuleReaderRequest + val reader = findModuleReader(req.scheme) + var spec: @Nullable ModuleReaderSpec? = null + if (reader != null) { + spec = reader.spec + } + transport.send(InitializeModuleReaderResponse(req.requestId, spec)) + } + Message.Type.INITIALIZE_RESOURCE_READER_REQUEST -> { + val req = msg as InitializeResourceReaderRequest + val reader = findResourceReader(req.scheme) + var spec: @Nullable ResourceReaderSpec? = null + if (reader != null) { + spec = reader.spec + } + transport.send(InitializeResourceReaderResponse(req.requestId, spec)) + } + Message.Type.LIST_MODULES_REQUEST -> { + val req = msg as ListModulesRequest + val reader = findModuleReader(req.uri.scheme) + if (reader == null) { + transport.send( + ListModulesResponse( + req.requestId, + req.evaluatorId, + null, + "No module reader found for scheme " + req.uri.scheme + ) + ) + return@start + } + try { + transport.send( + ListModulesResponse( + req.requestId, + req.evaluatorId, + reader.listElements(req.uri), + null + ) + ) + } catch (e: Exception) { + transport.send( + ListModulesResponse(req.requestId, req.evaluatorId, null, e.toString()) + ) + } + } + Message.Type.LIST_RESOURCES_REQUEST -> { + val req = msg as ListResourcesRequest + val reader = findModuleReader(req.uri.scheme) + if (reader == null) { + transport.send( + ListResourcesResponse( + req.requestId, + req.evaluatorId, + null, + "No resource reader found for scheme " + req.uri.scheme + ) + ) + return@start + } + try { + transport.send( + ListResourcesResponse( + req.requestId, + req.evaluatorId, + reader.listElements(req.uri), + null + ) + ) + } catch (e: Exception) { + transport.send( + ListResourcesResponse(req.requestId, req.evaluatorId, null, e.toString()) + ) + } + } + Message.Type.READ_MODULE_REQUEST -> { + val req = msg as ReadModuleRequest + val reader = findModuleReader(req.uri.scheme) + if (reader == null) { + transport.send( + ReadModuleResponse( + req.requestId, + req.evaluatorId, + null, + "No module reader found for scheme " + req.uri.scheme + ) + ) + return@start + } + try { + transport.send( + ReadModuleResponse(req.requestId, req.evaluatorId, reader.read(req.uri), null) + ) + } catch (e: Exception) { + transport.send(ReadModuleResponse(req.requestId, req.evaluatorId, null, e.toString())) + } + } + Message.Type.READ_RESOURCE_REQUEST -> { + val req = msg as ReadResourceRequest + val reader = findResourceReader(req.uri.scheme) + if (reader == null) { + transport.send( + ReadResourceResponse( + req.requestId, + req.evaluatorId, + byteArrayOf(), + "No resource reader found for scheme " + req.uri.scheme + ) + ) + return@start + } + try { + transport.send( + ReadResourceResponse(req.requestId, req.evaluatorId, reader.read(req.uri), null) + ) + } catch (e: Exception) { + transport.send( + ReadResourceResponse(req.requestId, req.evaluatorId, byteArrayOf(), e.toString()) + ) + } + } + else -> throw ProtocolException("Unexpected incoming request message: $msg") + } + } + ) + } +} diff --git a/pkl-core/src/test/kotlin/org/pkl/core/externalreader/ExternalResourceReader.kt b/pkl-core/src/test/kotlin/org/pkl/core/externalreader/ExternalResourceReader.kt new file mode 100644 index 000000000..bf5ab46f3 --- /dev/null +++ b/pkl-core/src/test/kotlin/org/pkl/core/externalreader/ExternalResourceReader.kt @@ -0,0 +1,27 @@ +/* + * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.core.externalreader + +import java.net.URI +import org.pkl.core.messaging.Messages.ResourceReaderSpec + +/** An external resource reader, to be used with [ExternalReaderRuntime]. */ +interface ExternalResourceReader : ExternalReaderBase { + fun read(uri: URI): ByteArray + + val spec: ResourceReaderSpec + get() = ResourceReaderSpec(scheme, hasHierarchicalUris, isGlobbable) +} diff --git a/pkl-core/src/test/kotlin/org/pkl/core/externalreader/TestExternalModuleReader.kt b/pkl-core/src/test/kotlin/org/pkl/core/externalreader/TestExternalModuleReader.kt new file mode 100644 index 000000000..12b543cec --- /dev/null +++ b/pkl-core/src/test/kotlin/org/pkl/core/externalreader/TestExternalModuleReader.kt @@ -0,0 +1,38 @@ +/* + * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.core.externalreader + +import java.net.URI +import org.pkl.core.module.PathElement + +class TestExternalModuleReader : ExternalModuleReader { + override val scheme: String = "test" + + override val hasHierarchicalUris: Boolean = false + + override val isLocal: Boolean = true + + override val isGlobbable: Boolean = false + + override fun read(uri: URI): String = + """ + name = "Pigeon" + age = 40 + """ + .trimIndent() + + override fun listElements(uri: URI): List = emptyList() +} diff --git a/pkl-core/src/test/kotlin/org/pkl/core/externalreader/TestExternalReaderProcess.kt b/pkl-core/src/test/kotlin/org/pkl/core/externalreader/TestExternalReaderProcess.kt new file mode 100644 index 000000000..5e19cf9c0 --- /dev/null +++ b/pkl-core/src/test/kotlin/org/pkl/core/externalreader/TestExternalReaderProcess.kt @@ -0,0 +1,130 @@ +/* + * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.core.externalreader + +import java.io.IOException +import java.io.PipedInputStream +import java.io.PipedOutputStream +import java.util.concurrent.CompletableFuture +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.ExecutionException +import java.util.concurrent.Future +import kotlin.random.Random +import org.pkl.core.externalreader.ExternalReaderMessages.* +import org.pkl.core.messaging.MessageTransport +import org.pkl.core.messaging.MessageTransports +import org.pkl.core.messaging.Messages.* +import org.pkl.core.messaging.ProtocolException + +class TestExternalReaderProcess(private val transport: MessageTransport) : ExternalReaderProcess { + private val initializeModuleReaderResponses: MutableMap> = + ConcurrentHashMap() + private val initializeResourceReaderResponses: MutableMap> = + ConcurrentHashMap() + + override fun close() { + transport.send(CloseExternalProcess()) + transport.close() + } + + override fun getTransport(): MessageTransport = transport + + fun run() { + try { + transport.start( + { throw ProtocolException("Unexpected incoming one-way message: $it") }, + { throw ProtocolException("Unexpected incoming request message: $it") }, + ) + } catch (e: ProtocolException) { + throw RuntimeException(e) + } catch (e: IOException) { + throw RuntimeException(e) + } + } + + override fun getModuleReaderSpec(scheme: String): ModuleReaderSpec? = + initializeModuleReaderResponses + .computeIfAbsent(scheme) { + CompletableFuture().apply { + val request = InitializeModuleReaderRequest(Random.nextLong(), scheme) + transport.send(request) { response -> + when (response) { + is InitializeModuleReaderResponse -> { + complete(response.spec) + } + else -> completeExceptionally(ProtocolException("unexpected response")) + } + } + } + } + .getUnderlying() + + override fun getResourceReaderSpec(scheme: String): ResourceReaderSpec? = + initializeResourceReaderResponses + .computeIfAbsent(scheme) { + CompletableFuture().apply { + val request = InitializeResourceReaderRequest(Random.nextLong(), scheme) + transport.send(request) { response -> + when (response) { + is InitializeResourceReaderResponse -> { + complete(response.spec) + } + else -> completeExceptionally(ProtocolException("unexpected response")) + } + } + } + } + .getUnderlying() + + companion object { + fun initializeTestHarness( + moduleReaders: List, + resourceReaders: List + ): Pair { + val rxIn = PipedInputStream(10240) + val rxOut = PipedOutputStream(rxIn) + val txIn = PipedInputStream(10240) + val txOut = PipedOutputStream(txIn) + val serverTransport = + MessageTransports.stream( + ExternalReaderMessagePackDecoder(rxIn), + ExternalReaderMessagePackEncoder(txOut), + {} + ) + val clientTransport = + MessageTransports.stream( + ExternalReaderMessagePackDecoder(txIn), + ExternalReaderMessagePackEncoder(rxOut), + {} + ) + + val runtime = ExternalReaderRuntime(moduleReaders, resourceReaders, clientTransport) + val proc = TestExternalReaderProcess(serverTransport) + + Thread(runtime::run).start() + Thread(proc::run).start() + + return proc to runtime + } + } +} + +fun Future.getUnderlying(): T = + try { + get() + } catch (e: ExecutionException) { + throw e.cause!! + } diff --git a/pkl-core/src/test/kotlin/org/pkl/core/externalreader/TestExternalResourceReader.kt b/pkl-core/src/test/kotlin/org/pkl/core/externalreader/TestExternalResourceReader.kt new file mode 100644 index 000000000..6cabf362b --- /dev/null +++ b/pkl-core/src/test/kotlin/org/pkl/core/externalreader/TestExternalResourceReader.kt @@ -0,0 +1,31 @@ +/* + * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.core.externalreader + +import java.net.URI +import org.pkl.core.module.PathElement + +class TestExternalResourceReader : ExternalResourceReader { + override val scheme: String = "test" + + override val hasHierarchicalUris: Boolean = false + + override val isGlobbable: Boolean = false + + override fun read(uri: URI): ByteArray = "success".toByteArray(Charsets.UTF_8) + + override fun listElements(uri: URI): List = emptyList() +} diff --git a/pkl-core/src/test/kotlin/org/pkl/core/messaging/BaseMessagePackCodecTest.kt b/pkl-core/src/test/kotlin/org/pkl/core/messaging/BaseMessagePackCodecTest.kt new file mode 100644 index 000000000..db95893c0 --- /dev/null +++ b/pkl-core/src/test/kotlin/org/pkl/core/messaging/BaseMessagePackCodecTest.kt @@ -0,0 +1,128 @@ +/* + * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.core.messaging + +import java.io.PipedInputStream +import java.io.PipedOutputStream +import java.net.URI +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows +import org.msgpack.core.MessagePack +import org.pkl.core.messaging.Messages.* +import org.pkl.core.module.PathElement + +class BaseMessagePackCodecTest { + private val encoder: MessageEncoder + private val decoder: MessageDecoder + + init { + val inputStream = PipedInputStream() + val outputStream = PipedOutputStream(inputStream) + encoder = BaseMessagePackEncoder(MessagePack.newDefaultPacker(outputStream)) + decoder = BaseMessagePackDecoder(MessagePack.newDefaultUnpacker(inputStream)) + } + + private fun roundtrip(message: Message) { + encoder.encode(message) + val decoded = decoder.decode() + assertThat(decoded).isEqualTo(message) + } + + @Test + fun `round-trip ReadResourceRequest`() { + roundtrip(ReadResourceRequest(123, 456, URI("some/resource.json"))) + } + + @Test + fun `round-trip ReadResourceResponse`() { + roundtrip(ReadResourceResponse(123, 456, byteArrayOf(1, 2, 3, 4, 5), null)) + } + + @Test + fun `round-trip ReadModuleRequest`() { + roundtrip(ReadModuleRequest(123, 456, URI("some/module.pkl"))) + } + + @Test + fun `round-trip ReadModuleResponse`() { + roundtrip(ReadModuleResponse(123, 456, "x = 42", null)) + } + + @Test + fun `round-trip ListModulesRequest`() { + roundtrip(ListModulesRequest(135, 246, URI("foo:/bar/baz/biz"))) + } + + @Test + fun `round-trip ListModulesResponse`() { + roundtrip( + ListModulesResponse( + 123, + 234, + listOf(PathElement("foo", true), PathElement("bar", false)), + null + ) + ) + roundtrip(ListModulesResponse(123, 234, null, "Something dun went wrong")) + } + + @Test + fun `round-trip ListResourcesRequest`() { + roundtrip(ListResourcesRequest(987, 1359, URI("bar:/bazzy"))) + } + + @Test + fun `round-trip ListResourcesResponse`() { + roundtrip( + ListResourcesResponse( + 3851, + 3019, + listOf(PathElement("foo", true), PathElement("bar", false)), + null + ) + ) + roundtrip(ListResourcesResponse(3851, 3019, null, "something went wrong")) + } + + @Test + fun `decode request with missing request ID`() { + val bytes = + MessagePack.newDefaultBufferPacker() + .apply { + packArrayHeader(2) + packInt(Message.Type.LIST_RESOURCES_REQUEST.code) + packMapHeader(1) + packString("uri") + packString("file:/test") + } + .toByteArray() + + val decoder = BaseMessagePackDecoder(MessagePack.newDefaultUnpacker(bytes)) + val exception = assertThrows { decoder.decode() } + assertThat(exception.message).contains("requestId") + } + + @Test + fun `decode invalid message header`() { + val bytes = MessagePack.newDefaultBufferPacker().apply { packInt(2) }.toByteArray() + + val decoder = BaseMessagePackDecoder(MessagePack.newDefaultUnpacker(bytes)) + val exception = assertThrows { decoder.decode() } + assertThat(exception).hasMessage("Malformed message header.") + assertThat(exception).hasRootCauseMessage("Expected Array, but got Integer (02)") + } +} diff --git a/pkl-core/src/test/kotlin/org/pkl/core/module/ModuleKeyFactoriesTest.kt b/pkl-core/src/test/kotlin/org/pkl/core/module/ModuleKeyFactoriesTest.kt index 055e79868..032fc5fe6 100644 --- a/pkl-core/src/test/kotlin/org/pkl/core/module/ModuleKeyFactoriesTest.kt +++ b/pkl-core/src/test/kotlin/org/pkl/core/module/ModuleKeyFactoriesTest.kt @@ -26,6 +26,7 @@ import org.pkl.commons.createParentDirectories import org.pkl.commons.toPath import org.pkl.commons.writeString import org.pkl.core.SecurityManagers +import org.pkl.core.externalreader.* class ModuleKeyFactoriesTest { @Test @@ -126,4 +127,23 @@ class ModuleKeyFactoriesTest { val module2 = factory.create(URI("other")) assertThat(module2).isNotPresent } + + @Test + fun externalProcess() { + val extReader = TestExternalModuleReader() + val (proc, runtime) = + TestExternalReaderProcess.initializeTestHarness(listOf(extReader), emptyList()) + + val factory = ModuleKeyFactories.externalProcess(extReader.scheme, proc) + + val module = factory.create(URI("test:foo")) + assertThat(module).isPresent + assertThat(module.get().uri.scheme).isEqualTo("test") + + val module2 = factory.create(URI("other")) + assertThat(module2).isNotPresent + + proc.close() + runtime.close() + } } diff --git a/pkl-core/src/test/kotlin/org/pkl/core/project/ProjectTest.kt b/pkl-core/src/test/kotlin/org/pkl/core/project/ProjectTest.kt index 5961375e5..6865c409e 100644 --- a/pkl-core/src/test/kotlin/org/pkl/core/project/ProjectTest.kt +++ b/pkl-core/src/test/kotlin/org/pkl/core/project/ProjectTest.kt @@ -70,6 +70,8 @@ class ProjectTest { listOf(path.resolve("modulepath1/"), path.resolve("modulepath2/")), Duration.ofMinutes(5.0), path, + null, + null, null ) val expectedAnnotations = diff --git a/pkl-core/src/test/kotlin/org/pkl/core/resource/ResourceReadersTest.kt b/pkl-core/src/test/kotlin/org/pkl/core/resource/ResourceReadersTest.kt index c5df4a41c..58c34b469 100644 --- a/pkl-core/src/test/kotlin/org/pkl/core/resource/ResourceReadersTest.kt +++ b/pkl-core/src/test/kotlin/org/pkl/core/resource/ResourceReadersTest.kt @@ -23,6 +23,8 @@ import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows import org.junit.jupiter.api.io.TempDir +import org.pkl.core.externalreader.TestExternalReaderProcess +import org.pkl.core.externalreader.TestExternalResourceReader import org.pkl.core.module.ModulePathResolver class ResourceReadersTest { @@ -132,4 +134,21 @@ class ResourceReadersTest { assertThat(resource).contains("success") } + + @Test + fun externalProcess() { + val extReader = TestExternalResourceReader() + val (proc, runtime) = + TestExternalReaderProcess.initializeTestHarness(emptyList(), listOf(extReader)) + + val reader = ResourceReaders.externalProcess(extReader.scheme, proc) + val resource = reader.read(URI("test:foo")) + + assertThat(resource).isPresent + assertThat(resource.get()).isInstanceOf(Resource::class.java) + assertThat((resource.get() as Resource).text).contains("success") + + proc.close() + runtime.close() + } } diff --git a/pkl-core/src/test/resources/org/pkl/core/project/project1/PklProject b/pkl-core/src/test/resources/org/pkl/core/project/project1/PklProject index 266ba3da7..35c76e480 100644 --- a/pkl-core/src/test/resources/org/pkl/core/project/project1/PklProject +++ b/pkl-core/src/test/resources/org/pkl/core/project/project1/PklProject @@ -22,4 +22,22 @@ evaluatorSettings { noCache = false rootDir = "my-root-dir/" timeout = 5.min + externalModuleReaders { + ["scheme1"] { + executable = "reader1" + } + ["scheme2"] { + executable = "reader2" + arguments { "with"; "args" } + } + } + externalResourceReaders { + ["scheme3"] { + executable = "reader3" + } + ["scheme4"] { + executable = "reader4" + arguments { "with"; "args" } + } + } } diff --git a/pkl-doc/gradle.lockfile b/pkl-doc/gradle.lockfile index a79cfec2f..3daeb0ab2 100644 --- a/pkl-doc/gradle.lockfile +++ b/pkl-doc/gradle.lockfile @@ -75,6 +75,7 @@ org.junit.jupiter:junit-jupiter-params:5.11.2=testCompileClasspath,testImplement org.junit.platform:junit-platform-commons:1.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata org.junit.platform:junit-platform-engine:1.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata org.junit:junit-bom:5.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata +org.msgpack:msgpack-core:0.9.8=runtimeClasspath,testRuntimeClasspath org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata org.organicdesign:Paguro:3.10.3=runtimeClasspath,testRuntimeClasspath org.snakeyaml:snakeyaml-engine:2.5=runtimeClasspath,testRuntimeClasspath diff --git a/pkl-doc/src/main/kotlin/org/pkl/doc/CliDocGenerator.kt b/pkl-doc/src/main/kotlin/org/pkl/doc/CliDocGenerator.kt index 4ae7605f7..499c6ba25 100644 --- a/pkl-doc/src/main/kotlin/org/pkl/doc/CliDocGenerator.kt +++ b/pkl-doc/src/main/kotlin/org/pkl/doc/CliDocGenerator.kt @@ -24,8 +24,8 @@ import org.pkl.commons.cli.CliCommand import org.pkl.commons.cli.CliException import org.pkl.commons.toPath import org.pkl.core.* -import org.pkl.core.module.ModuleKeyFactories import org.pkl.core.packages.* +import org.pkl.core.util.Readers /** * Entry point for the high-level Pkldoc API. @@ -250,7 +250,8 @@ class CliDocGenerator(private val options: CliDocGeneratorOptions) : CliCommand( importedModules[pklBaseUri] = evaluator.evaluateSchema(ModuleSource.uri(pklBaseUri)) } } finally { - ModuleKeyFactories.closeQuietly(builder.moduleKeyFactories) + Readers.closeQuietly(builder.moduleKeyFactories) + Readers.closeQuietly(builder.resourceReaders) } val versions = mutableMapOf() diff --git a/pkl-executor/gradle.lockfile b/pkl-executor/gradle.lockfile index 891b28238..05dc497dc 100644 --- a/pkl-executor/gradle.lockfile +++ b/pkl-executor/gradle.lockfile @@ -29,6 +29,7 @@ org.junit.jupiter:junit-jupiter-params:5.11.2=testCompileClasspath,testImplement org.junit.platform:junit-platform-commons:1.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata org.junit.platform:junit-platform-engine:1.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata org.junit:junit-bom:5.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata +org.msgpack:msgpack-core:0.9.8=testRuntimeClasspath org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata org.organicdesign:Paguro:3.10.3=testRuntimeClasspath org.pkl-lang:pkl-config-java-all:0.25.0=pklHistoricalDistributions diff --git a/pkl-gradle/src/main/java/org/pkl/gradle/task/BasePklTask.java b/pkl-gradle/src/main/java/org/pkl/gradle/task/BasePklTask.java index 1b9e17815..fdd7c2c20 100644 --- a/pkl-gradle/src/main/java/org/pkl/gradle/task/BasePklTask.java +++ b/pkl-gradle/src/main/java/org/pkl/gradle/task/BasePklTask.java @@ -25,6 +25,7 @@ import java.time.Duration; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.function.Function; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -170,7 +171,9 @@ protected CliBaseOptions getCliBaseOptions() { getTestPort().getOrElse(-1), Collections.emptyList(), getHttpProxy().getOrNull(), - getHttpNoProxy().getOrElse(List.of())); + getHttpNoProxy().getOrElse(List.of()), + Map.of(), + Map.of()); } return cachedOptions; } diff --git a/pkl-gradle/src/main/java/org/pkl/gradle/task/ModulesTask.java b/pkl-gradle/src/main/java/org/pkl/gradle/task/ModulesTask.java index f7c2b905b..59a5ea971 100644 --- a/pkl-gradle/src/main/java/org/pkl/gradle/task/ModulesTask.java +++ b/pkl-gradle/src/main/java/org/pkl/gradle/task/ModulesTask.java @@ -182,7 +182,9 @@ protected CliBaseOptions getCliBaseOptions() { getTestPort().getOrElse(-1), Collections.emptyList(), null, - List.of()); + List.of(), + Map.of(), + Map.of()); } return cachedOptions; } diff --git a/pkl-server/src/main/kotlin/org/pkl/server/ClientLogger.kt b/pkl-server/src/main/kotlin/org/pkl/server/ClientLogger.kt index babcad0d8..cf224e0f8 100644 --- a/pkl-server/src/main/kotlin/org/pkl/server/ClientLogger.kt +++ b/pkl-server/src/main/kotlin/org/pkl/server/ClientLogger.kt @@ -17,16 +17,17 @@ package org.pkl.server import org.pkl.core.Logger import org.pkl.core.StackFrame +import org.pkl.core.messaging.MessageTransport internal class ClientLogger( private val evaluatorId: Long, private val transport: MessageTransport ) : Logger { override fun trace(message: String, frame: StackFrame) { - transport.send(LogMessage(evaluatorId, level = 0, message, frame.moduleUri)) + transport.send(LogMessage(evaluatorId, 0, message, frame.moduleUri)) } override fun warn(message: String, frame: StackFrame) { - transport.send(LogMessage(evaluatorId, level = 1, message, frame.moduleUri)) + transport.send(LogMessage(evaluatorId, 1, message, frame.moduleUri)) } } diff --git a/pkl-server/src/main/kotlin/org/pkl/server/ClientModuleKeyFactory.kt b/pkl-server/src/main/kotlin/org/pkl/server/ClientModuleKeyFactory.kt index b0b8ab77b..b53447b09 100644 --- a/pkl-server/src/main/kotlin/org/pkl/server/ClientModuleKeyFactory.kt +++ b/pkl-server/src/main/kotlin/org/pkl/server/ClientModuleKeyFactory.kt @@ -15,136 +15,27 @@ */ package org.pkl.server -import java.io.IOException import java.net.URI import java.util.Optional -import java.util.concurrent.CompletableFuture -import java.util.concurrent.ConcurrentHashMap -import java.util.concurrent.Future -import kotlin.random.Random -import org.pkl.core.SecurityManager -import org.pkl.core.module.ModuleKey -import org.pkl.core.module.ModuleKeyFactory -import org.pkl.core.module.PathElement -import org.pkl.core.module.ResolvedModuleKey -import org.pkl.core.module.ResolvedModuleKeys +import org.pkl.core.messaging.* +import org.pkl.core.messaging.Messages.* +import org.pkl.core.module.* +import org.pkl.core.module.ExternalModuleResolver internal class ClientModuleKeyFactory( private val readerSpecs: Collection, transport: MessageTransport, evaluatorId: Long ) : ModuleKeyFactory { - companion object { - private class ClientModuleKeyResolver( - private val transport: MessageTransport, - private val evaluatorId: Long, - ) { - private val readResponses: MutableMap> = ConcurrentHashMap() - - private val listResponses: MutableMap>> = ConcurrentHashMap() - - fun listElements(securityManager: SecurityManager, uri: URI): List { - securityManager.checkResolveModule(uri) - return doListElements(uri) - } - - fun hasElement(securityManager: SecurityManager, uri: URI): Boolean { - securityManager.checkResolveModule(uri) - return try { - doReadModule(uri) - true - } catch (e: IOException) { - false - } - } - - fun resolveModule(securityManager: SecurityManager, uri: URI): String { - securityManager.checkResolveModule(uri) - return doReadModule(uri) - } - - private fun doReadModule(uri: URI): String = - readResponses - .computeIfAbsent(uri) { - CompletableFuture().apply { - val request = ReadModuleRequest(Random.nextLong(), evaluatorId, uri) - transport.send(request) { response -> - when (response) { - is ReadModuleResponse -> { - if (response.error != null) { - completeExceptionally(IOException(response.error)) - } else { - complete(response.contents ?: "") - } - } - else -> { - completeExceptionally(ProtocolException("unexpected response")) - } - } - } - } - } - .getUnderlying() - - private fun doListElements(uri: URI): List = - listResponses - .computeIfAbsent(uri) { - CompletableFuture>().apply { - val request = ListModulesRequest(Random.nextLong(), evaluatorId, uri) - transport.send(request) { response -> - when (response) { - is ListModulesResponse -> { - if (response.error != null) { - completeExceptionally(IOException(response.error)) - } else { - complete(response.pathElements ?: emptyList()) - } - } - else -> completeExceptionally(ProtocolException("unexpected response")) - } - } - } - } - .getUnderlying() - } - - /** [ModuleKey] that delegates module reads to the client. */ - private class ClientModuleKey( - private val uri: URI, - private val spec: ModuleReaderSpec, - private val resolver: ClientModuleKeyResolver, - ) : ModuleKey { - override fun isLocal(): Boolean = spec.isLocal - - override fun hasHierarchicalUris(): Boolean = spec.hasHierarchicalUris - - override fun isGlobbable(): Boolean = spec.isGlobbable - - override fun getUri(): URI = uri - - override fun listElements(securityManager: SecurityManager, baseUri: URI): List = - resolver.listElements(securityManager, baseUri) - - override fun resolve(securityManager: SecurityManager): ResolvedModuleKey { - val contents = resolver.resolveModule(securityManager, uri) - return ResolvedModuleKeys.virtual(this, uri, contents, true) - } - - override fun hasElement(securityManager: SecurityManager, uri: URI): Boolean { - return resolver.hasElement(securityManager, uri) - } - } - } - private val schemes = readerSpecs.map { it.scheme } - private val resolver: ClientModuleKeyResolver = ClientModuleKeyResolver(transport, evaluatorId) + private val resolver: ExternalModuleResolver = ExternalModuleResolver(transport, evaluatorId) override fun create(uri: URI): Optional = when (uri.scheme) { in schemes -> { val readerSpec = readerSpecs.find { it.scheme == uri.scheme }!! - val moduleKey = ClientModuleKey(uri, readerSpec, resolver) + val moduleKey = ModuleKeys.externalResolver(uri, readerSpec, resolver) Optional.of(moduleKey) } else -> Optional.empty() diff --git a/pkl-server/src/main/kotlin/org/pkl/server/ClientResourceReader.kt b/pkl-server/src/main/kotlin/org/pkl/server/ClientResourceReader.kt deleted file mode 100644 index 339375492..000000000 --- a/pkl-server/src/main/kotlin/org/pkl/server/ClientResourceReader.kt +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.pkl.server - -import java.io.IOException -import java.net.URI -import java.util.Optional -import java.util.concurrent.CompletableFuture -import java.util.concurrent.ConcurrentHashMap -import java.util.concurrent.Future -import kotlin.random.Random -import org.pkl.core.SecurityManager -import org.pkl.core.module.PathElement -import org.pkl.core.resource.Resource -import org.pkl.core.resource.ResourceReader - -/** Resource reader that delegates read logic to the client. */ -internal class ClientResourceReader( - private val transport: MessageTransport, - private val evaluatorId: Long, - private val readerSpec: ResourceReaderSpec, -) : ResourceReader { - private val readResponses: MutableMap> = ConcurrentHashMap() - - private val listResources: MutableMap>> = ConcurrentHashMap() - - override fun hasHierarchicalUris(): Boolean = readerSpec.hasHierarchicalUris - - override fun isGlobbable(): Boolean = readerSpec.isGlobbable - - override fun getUriScheme() = readerSpec.scheme - - override fun read(uri: URI): Optional = Optional.of(Resource(uri, doRead(uri))) - - override fun hasElement(securityManager: SecurityManager, elementUri: URI): Boolean { - securityManager.checkResolveResource(elementUri) - return try { - doRead(elementUri) - true - } catch (e: IOException) { - false - } - } - - override fun listElements(securityManager: SecurityManager, baseUri: URI): List { - securityManager.checkResolveResource(baseUri) - return doListElements(baseUri) - } - - private fun doListElements(baseUri: URI): List = - listResources - .computeIfAbsent(baseUri) { - CompletableFuture>().apply { - val request = ListResourcesRequest(Random.nextLong(), evaluatorId, baseUri) - transport.send(request) { response -> - when (response) { - is ListResourcesResponse -> - if (response.error != null) { - completeExceptionally(IOException(response.error)) - } else { - complete(response.pathElements ?: emptyList()) - } - else -> completeExceptionally(ProtocolException("Unexpected response")) - } - } - } - } - .getUnderlying() - - private fun doRead(uri: URI): ByteArray = - readResponses - .computeIfAbsent(uri) { - CompletableFuture().apply { - val request = ReadResourceRequest(Random.nextLong(), evaluatorId, uri) - transport.send(request) { response -> - when (response) { - is ReadResourceResponse -> { - if (response.error != null) { - completeExceptionally(IOException(response.error)) - } else { - complete(response.contents ?: ByteArray(0)) - } - } - else -> { - completeExceptionally(ProtocolException("Unexpected response: $response")) - } - } - } - } - } - .getUnderlying() -} diff --git a/pkl-server/src/main/kotlin/org/pkl/server/MessageDecoders.kt b/pkl-server/src/main/kotlin/org/pkl/server/MessageDecoders.kt deleted file mode 100644 index 416d709cc..000000000 --- a/pkl-server/src/main/kotlin/org/pkl/server/MessageDecoders.kt +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.pkl.server - -import java.io.InputStream -import org.msgpack.core.MessagePack -import org.msgpack.core.MessageUnpacker - -/** Factory methods for creating [MessageDecoder]s. */ -internal object MessageDecoders { - fun from(stream: InputStream): MessageDecoder = - MessagePackDecoder(MessagePack.newDefaultUnpacker(stream)) - - fun from(unpacker: MessageUnpacker): MessageDecoder = MessagePackDecoder(unpacker) - - fun from(array: ByteArray): MessageDecoder = - MessagePackDecoder(MessagePack.newDefaultUnpacker(array)) -} diff --git a/pkl-server/src/main/kotlin/org/pkl/server/MessagePackDecoder.kt b/pkl-server/src/main/kotlin/org/pkl/server/MessagePackDecoder.kt deleted file mode 100644 index 076cb6b27..000000000 --- a/pkl-server/src/main/kotlin/org/pkl/server/MessagePackDecoder.kt +++ /dev/null @@ -1,292 +0,0 @@ -/* - * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.pkl.server - -import java.net.URI -import java.nio.file.Path -import java.time.Duration -import java.util.regex.Pattern -import org.msgpack.core.MessageTypeException -import org.msgpack.core.MessageUnpacker -import org.msgpack.value.Value -import org.msgpack.value.impl.ImmutableStringValueImpl -import org.pkl.core.evaluatorSettings.PklEvaluatorSettings -import org.pkl.core.module.PathElement -import org.pkl.core.packages.Checksums - -internal class MessagePackDecoder(private val unpacker: MessageUnpacker) : MessageDecoder { - override fun decode(): Message? { - if (!unpacker.hasNext()) return null - - val code = - try { - val arraySize = unpacker.unpackArrayHeader() - if (arraySize != 2) { - throw DecodeException("Malformed message header (expected size 2, but got $arraySize).") - } - unpacker.unpackInt() - } catch (e: MessageTypeException) { - throw DecodeException("Malformed message header.", e) - } - - return try { - val map = unpacker.unpackValue().asMapValue().map() - when (code) { - MessageType.CREATE_EVALUATOR_REQUEST.code -> { - CreateEvaluatorRequest( - requestId = map.get("requestId").asIntegerValue().asLong(), - allowedModules = map.unpackStringListOrNull("allowedModules")?.map(Pattern::compile), - allowedResources = - map.unpackStringListOrNull("allowedResources")?.map(Pattern::compile), - clientModuleReaders = map.unpackModuleReaderSpec(), - clientResourceReaders = map.unpackResourceReaderSpec(), - modulePaths = map.unpackStringListOrNull("modulePaths")?.map(Path::of), - env = map.unpackStringMapOrNull("env"), - properties = map.unpackStringMapOrNull("properties"), - timeout = map.unpackLongOrNull("timeoutSeconds")?.let(Duration::ofSeconds), - rootDir = map.unpackStringOrNull("rootDir")?.let(Path::of), - cacheDir = map.unpackStringOrNull("cacheDir")?.let(Path::of), - outputFormat = map.unpackStringOrNull("outputFormat"), - project = map.unpackProject(), - http = map.unpackHttp(), - ) - } - MessageType.CREATE_EVALUATOR_RESPONSE.code -> { - CreateEvaluatorResponse( - requestId = map.unpackLong("requestId"), - evaluatorId = map.unpackLongOrNull("evaluatorId"), - error = map.unpackStringOrNull("error") - ) - } - MessageType.CLOSE_EVALUATOR.code -> { - CloseEvaluator(evaluatorId = map.unpackLong("evaluatorId")) - } - MessageType.EVALUATE_REQUEST.code -> { - EvaluateRequest( - requestId = map.unpackLong("requestId"), - evaluatorId = map.unpackLong("evaluatorId"), - moduleUri = map.unpackString("moduleUri").let(::URI), - moduleText = map.unpackStringOrNull("moduleText"), - expr = map.unpackStringOrNull("expr") - ) - } - MessageType.EVALUATE_RESPONSE.code -> { - EvaluateResponse( - requestId = map.unpackLong("requestId"), - evaluatorId = map.unpackLong("evaluatorId"), - result = map.unpackByteArrayOrNull("result"), - error = map.unpackStringOrNull("error") - ) - } - MessageType.LOG_MESSAGE.code -> { - LogMessage( - evaluatorId = map.unpackLong("evaluatorId"), - level = map.unpackIntValue("level"), - message = map.unpackString("message"), - frameUri = map.unpackString("frameUri") - ) - } - MessageType.READ_RESOURCE_REQUEST.code -> { - ReadResourceRequest( - requestId = map.unpackLong("requestId"), - evaluatorId = map.unpackLong("evaluatorId"), - uri = map.unpackString("uri").let(::URI) - ) - } - MessageType.READ_RESOURCE_RESPONSE.code -> { - ReadResourceResponse( - requestId = map.unpackLong("requestId"), - evaluatorId = map.unpackLong("evaluatorId"), - contents = map.unpackByteArrayOrNull("contents"), - error = map.unpackStringOrNull("error") - ) - } - MessageType.READ_MODULE_REQUEST.code -> { - ReadModuleRequest( - requestId = map.unpackLong("requestId"), - evaluatorId = map.unpackLong("evaluatorId"), - uri = map.unpackString("uri").let(::URI) - ) - } - MessageType.READ_MODULE_RESPONSE.code -> { - ReadModuleResponse( - requestId = map.unpackLong("requestId"), - evaluatorId = map.unpackLong("evaluatorId"), - contents = map.unpackStringOrNull("contents"), - error = map.unpackStringOrNull("error") - ) - } - MessageType.LIST_MODULES_REQUEST.code -> { - ListModulesRequest( - requestId = map.unpackLong("requestId"), - evaluatorId = map.unpackLong("evaluatorId"), - uri = map.unpackString("uri").let(::URI) - ) - } - MessageType.LIST_MODULES_RESPONSE.code -> { - ListModulesResponse( - requestId = map.unpackLong("requestId"), - evaluatorId = map.unpackLong("evaluatorId"), - pathElements = map.unpackPathElements("pathElements"), - error = map.unpackStringOrNull("error") - ) - } - MessageType.LIST_RESOURCES_REQUEST.code -> { - ListResourcesRequest( - requestId = map.unpackLong("requestId"), - evaluatorId = map.unpackLong("evaluatorId"), - uri = map.unpackString("uri").let(::URI) - ) - } - MessageType.LIST_RESOURCES_RESPONSE.code -> { - ListResourcesResponse( - requestId = map.unpackLong("requestId"), - evaluatorId = map.unpackLong("evaluatorId"), - pathElements = map.unpackPathElements("pathElements"), - error = map.unpackStringOrNull("error") - ) - } - else -> throw ProtocolException("Invalid message code: $code") - } - } catch (e: MessageTypeException) { - throw DecodeException("Malformed message body for message with code `$code`.", e) - } - } - - private fun Array.unpackValueOrNull(key: String): Value? { - for (i in indices.step(2)) { - val currKey = this[i].asStringValue().asString() - if (currKey == key) return this[i + 1] - } - return null - } - - private fun Map.getNullable(key: String): Value? = - this[ImmutableStringValueImpl(key)] - - private fun Map.get(key: String): Value = - getNullable(key) ?: throw DecodeException("Missing message parameter `$key`") - - private fun Array.unpackValue(key: String): Value = - unpackValueOrNull(key) ?: throw DecodeException("Missing message parameter `$key`.") - - private fun Map.unpackStringListOrNull(key: String): List? { - val value = getNullable(key) ?: return null - return value.asArrayValue().map { it.asStringValue().asString() } - } - - private fun Map.unpackStringMapOrNull(key: String): Map? { - val value = getNullable(key) ?: return null - return value.asMapValue().entrySet().associate { (k, v) -> - k.asStringValue().asString() to v.asStringValue().asString() - } - } - - private fun Map.unpackLong(key: String): Long = get(key).asIntegerValue().asLong() - - private fun Map.unpackBoolean(key: String): Boolean = - get(key).asBooleanValue().boolean - - private fun Map.unpackBooleanOrNull(key: String): Boolean? = - getNullable(key)?.asBooleanValue()?.boolean - - private fun Map.unpackLongOrNull(key: String): Long? = - getNullable(key)?.asIntegerValue()?.asLong() - - private fun Map.unpackIntValue(key: String): Int = get(key).asIntegerValue().asInt() - - private fun Map.unpackString(key: String): String = - get(key).asStringValue().asString() - - private fun Map.unpackStringOrNull(key: String): String? = - getNullable(key)?.asStringValue()?.asString() - - private fun Map.unpackByteArrayOrNull(key: String): ByteArray? = - getNullable(key)?.asBinaryValue()?.asByteArray() - - private fun Map.unpackPathElements(key: String): List? = - getNullable(key)?.asArrayValue()?.map { pathElement -> - val map = pathElement.asMapValue().map() - PathElement(map.unpackString("name"), map.unpackBoolean("isDirectory")) - } - - private fun Map.unpackModuleReaderSpec(): List? { - val keys = getNullable("clientModuleReaders") ?: return null - return keys.asArrayValue().toList().map { value -> - val readerMap = value.asMapValue().map() - ModuleReaderSpec( - scheme = readerMap.unpackString("scheme"), - hasHierarchicalUris = readerMap.unpackBoolean("hasHierarchicalUris"), - isLocal = readerMap.unpackBoolean("isLocal"), - isGlobbable = readerMap.unpackBoolean("isGlobbable") - ) - } - } - - private fun Map.unpackResourceReaderSpec(): List { - val keys = getNullable("clientResourceReaders") ?: return emptyList() - return keys.asArrayValue().toList().map { value -> - val readerMap = value.asMapValue().map() - ResourceReaderSpec( - scheme = readerMap.unpackString("scheme"), - hasHierarchicalUris = readerMap.unpackBoolean("hasHierarchicalUris"), - isGlobbable = readerMap.unpackBoolean("isGlobbable") - ) - } - } - - private fun Map.unpackProject(): Project? { - val projMap = getNullable("project")?.asMapValue()?.map() ?: return null - val projectFileUri = URI(projMap.unpackString("projectFileUri")) - val dependencies = projMap.unpackDependencies("dependencies") - return Project(projectFileUri, null, dependencies) - } - - private fun Map.unpackHttp(): Http? { - val httpMap = getNullable("http")?.asMapValue()?.map() ?: return null - val proxy = httpMap.unpackProxy() - val caCertificates = httpMap.getNullable("caCertificates")?.asBinaryValue()?.asByteArray() - return Http(caCertificates, proxy) - } - - private fun Map.unpackProxy(): PklEvaluatorSettings.Proxy? { - val proxyMap = getNullable("proxy")?.asMapValue()?.map() ?: return null - val address = proxyMap.unpackString("address") - val noProxy = proxyMap.unpackStringListOrNull("noProxy") - return PklEvaluatorSettings.Proxy.create(address, noProxy) - } - - private fun Map.unpackDependencies(name: String): Map { - val mapValue = get(name).asMapValue().map() - return mapValue.entries.associate { (key, value) -> - val dependencyName = key.asStringValue().asString() - val dependencyObj = value.asMapValue().map() - val type = dependencyObj.unpackString("type") - val packageUri = URI(dependencyObj.unpackString("packageUri")) - if (type == DependencyType.REMOTE.value) { - val checksums = - dependencyObj.getNullable("checksums")?.asMapValue()?.map()?.let { obj -> - val sha256 = obj.unpackString("sha256") - Checksums(sha256) - } - return@associate dependencyName to RemoteDependency(packageUri, checksums) - } - val dependencies = dependencyObj.unpackDependencies("dependencies") - val projectFileUri = dependencyObj.unpackString("projectFileUri") - dependencyName to Project(URI(projectFileUri), packageUri, dependencies) - } - } -} diff --git a/pkl-server/src/main/kotlin/org/pkl/server/MessagePackEncoder.kt b/pkl-server/src/main/kotlin/org/pkl/server/MessagePackEncoder.kt deleted file mode 100644 index 3b3a2b40d..000000000 --- a/pkl-server/src/main/kotlin/org/pkl/server/MessagePackEncoder.kt +++ /dev/null @@ -1,332 +0,0 @@ -/* - * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.pkl.server - -import kotlin.io.path.pathString -import org.msgpack.core.MessagePacker -import org.pkl.core.module.PathElement -import org.pkl.core.packages.Checksums - -internal class MessagePackEncoder(private val packer: MessagePacker) : MessageEncoder { - private fun MessagePacker.packModuleReaderSpec(reader: ModuleReaderSpec) { - packMapHeader(4) - packKeyValue("scheme", reader.scheme) - packKeyValue("hasHierarchicalUris", reader.hasHierarchicalUris) - packKeyValue("isLocal", reader.isLocal) - packKeyValue("isGlobbable", reader.isGlobbable) - } - - private fun MessagePacker.packResourceReaderSpec(reader: ResourceReaderSpec) { - packMapHeader(3) - packKeyValue("scheme", reader.scheme) - packKeyValue("hasHierarchicalUris", reader.hasHierarchicalUris) - packKeyValue("isGlobbable", reader.isGlobbable) - } - - private fun MessagePacker.packPathElement(pathElement: PathElement) { - packMapHeader(2) - packKeyValue("name", pathElement.name) - packKeyValue("isDirectory", pathElement.isDirectory) - } - - private fun MessagePacker.packProject(project: Project) { - packMapHeader(2) - packKeyValue("projectFileUri", project.projectFileUri.toString()) - packString("dependencies") - packDependencies(project.dependencies) - } - - private fun MessagePacker.packHttp(http: Http) { - if ((http.caCertificates ?: http.proxy) == null) { - packMapHeader(0) - return - } - packMapHeader(0, http.caCertificates, http.proxy) - packKeyValue("caCertificates", http.caCertificates) - http.proxy?.let { proxy -> - packString("proxy") - packMapHeader(0, proxy.address, proxy.noProxy) - packKeyValue("address", proxy.address?.toString()) - packKeyValue("noProxy", proxy.noProxy) - } - } - - private fun MessagePacker.packDependencies(dependencies: Map) { - packMapHeader(dependencies.size) - for ((name, dep) in dependencies) { - packString(name) - if (dep is Project) { - packMapHeader(4) - packKeyValue("type", dep.type.value) - packKeyValue("packageUri", dep.packageUri.toString()) - packKeyValue("projectFileUri", dep.projectFileUri.toString()) - packString("dependencies") - packDependencies(dep.dependencies) - } else { - dep as RemoteDependency - packMapHeader(dep.checksums?.let { 3 } ?: 2) - packKeyValue("type", dep.type.value) - packKeyValue("packageUri", dep.packageUri.toString()) - dep.checksums?.let { checksums -> - packString("checksums") - packChecksums(checksums) - } - } - } - } - - private fun MessagePacker.packChecksums(checksums: Checksums) { - packMapHeader(1) - packKeyValue("sha256", checksums.sha256) - } - - override fun encode(msg: Message) = - with(packer) { - packArrayHeader(2) - packInt(msg.type.code) - - @Suppress("DuplicatedCode") - when (msg.type.code) { - MessageType.CREATE_EVALUATOR_REQUEST.code -> { - msg as CreateEvaluatorRequest - packMapHeader( - 8, - msg.timeout, - msg.rootDir, - msg.cacheDir, - msg.outputFormat, - msg.project, - msg.http - ) - packKeyValue("requestId", msg.requestId) - packKeyValue("allowedModules", msg.allowedModules?.map { it.toString() }) - packKeyValue("allowedResources", msg.allowedResources?.map { it.toString() }) - if (msg.clientModuleReaders != null) { - packString("clientModuleReaders") - packArrayHeader(msg.clientModuleReaders.size) - for (moduleReader in msg.clientModuleReaders) { - packModuleReaderSpec(moduleReader) - } - } - if (msg.clientResourceReaders != null) { - packString("clientResourceReaders") - packArrayHeader(msg.clientResourceReaders.size) - for (resourceReader in msg.clientResourceReaders) { - packResourceReaderSpec(resourceReader) - } - } - packKeyValue("modulePaths", msg.modulePaths?.map { it.pathString }) - packKeyValue("env", msg.env) - packKeyValue("properties", msg.properties) - packKeyValue("timeoutSeconds", msg.timeout?.toSeconds()) - packKeyValue("rootDir", msg.rootDir?.pathString) - packKeyValue("cacheDir", msg.cacheDir?.pathString) - packKeyValue("outputFormat", msg.outputFormat) - if (msg.project != null) { - packString("project") - packProject(msg.project) - } - if (msg.http != null) { - packString("http") - packHttp(msg.http) - } - } - MessageType.CREATE_EVALUATOR_RESPONSE.code -> { - msg as CreateEvaluatorResponse - packMapHeader(1, msg.evaluatorId, msg.error) - packKeyValue("requestId", msg.requestId) - packKeyValue("evaluatorId", msg.evaluatorId) - packKeyValue("error", msg.error) - } - MessageType.CLOSE_EVALUATOR.code -> { - msg as CloseEvaluator - packMapHeader(1) - packKeyValue("evaluatorId", msg.evaluatorId) - } - MessageType.EVALUATE_REQUEST.code -> { - msg as EvaluateRequest - packMapHeader(3, msg.moduleText, msg.expr) - packKeyValue("requestId", msg.requestId) - packKeyValue("evaluatorId", msg.evaluatorId) - packKeyValue("moduleUri", msg.moduleUri.toString()) - packKeyValue("moduleText", msg.moduleText) - packKeyValue("expr", msg.expr) - } - MessageType.EVALUATE_RESPONSE.code -> { - msg as EvaluateResponse - packMapHeader(2, msg.result, msg.error) - packKeyValue("requestId", msg.requestId) - packKeyValue("evaluatorId", msg.evaluatorId) - packKeyValue("result", msg.result) - packKeyValue("error", msg.error) - } - MessageType.LOG_MESSAGE.code -> { - msg as LogMessage - packMapHeader(4) - packKeyValue("evaluatorId", msg.evaluatorId) - packKeyValue("level", msg.level) - packKeyValue("message", msg.message) - packKeyValue("frameUri", msg.frameUri) - } - MessageType.READ_RESOURCE_REQUEST.code -> { - msg as ReadResourceRequest - packMapHeader(3) - packKeyValue("requestId", msg.requestId) - packKeyValue("evaluatorId", msg.evaluatorId) - packKeyValue("uri", msg.uri.toString()) - } - MessageType.READ_RESOURCE_RESPONSE.code -> { - msg as ReadResourceResponse - packMapHeader(2, msg.contents, msg.error) - packKeyValue("requestId", msg.requestId) - packKeyValue("evaluatorId", msg.evaluatorId) - packKeyValue("contents", msg.contents) - packKeyValue("error", msg.error) - } - MessageType.READ_MODULE_REQUEST.code -> { - msg as ReadModuleRequest - packMapHeader(3) - packKeyValue("requestId", msg.requestId) - packKeyValue("evaluatorId", msg.evaluatorId) - packKeyValue("uri", msg.uri.toString()) - } - MessageType.READ_MODULE_RESPONSE.code -> { - msg as ReadModuleResponse - packMapHeader(2, msg.contents, msg.error) - packKeyValue("requestId", msg.requestId) - packKeyValue("evaluatorId", msg.evaluatorId) - packKeyValue("contents", msg.contents) - packKeyValue("error", msg.error) - } - MessageType.LIST_MODULES_REQUEST.code -> { - msg as ListModulesRequest - packMapHeader(3) - packKeyValue("requestId", msg.requestId) - packKeyValue("evaluatorId", msg.evaluatorId) - packKeyValue("uri", msg.uri.toString()) - } - MessageType.LIST_MODULES_RESPONSE.code -> { - msg as ListModulesResponse - packMapHeader(2, msg.pathElements, msg.error) - packKeyValue("requestId", msg.requestId) - packKeyValue("evaluatorId", msg.evaluatorId) - if (msg.pathElements != null) { - packString("pathElements") - packArrayHeader(msg.pathElements.size) - for (pathElement in msg.pathElements) { - packPathElement(pathElement) - } - } - packKeyValue("error", msg.error) - } - MessageType.LIST_RESOURCES_REQUEST.code -> { - msg as ListResourcesRequest - packMapHeader(3) - packKeyValue("requestId", msg.requestId) - packKeyValue("evaluatorId", msg.evaluatorId) - packKeyValue("uri", msg.uri.toString()) - } - MessageType.LIST_RESOURCES_RESPONSE.code -> { - msg as ListResourcesResponse - packMapHeader(2, msg.pathElements, msg.error) - packKeyValue("requestId", msg.requestId) - packKeyValue("evaluatorId", msg.evaluatorId) - if (msg.pathElements != null) { - packString("pathElements") - packArrayHeader(msg.pathElements.size) - for (pathElement in msg.pathElements) { - packPathElement(pathElement) - } - } - packKeyValue("error", msg.error) - } - else -> { - throw RuntimeException("Missing encoding for ${msg.javaClass.simpleName}") - } - } - - flush() - } - - private fun MessagePacker.packMapHeader(size: Int, value1: Any?, value2: Any?) = - packMapHeader(size + (if (value1 != null) 1 else 0) + (if (value2 != null) 1 else 0)) - - private fun MessagePacker.packMapHeader( - size: Int, - value1: Any?, - value2: Any?, - value3: Any?, - value4: Any?, - value5: Any?, - value6: Any? - ) = - packMapHeader( - size + - (if (value1 != null) 1 else 0) + - (if (value2 != null) 1 else 0) + - (if (value3 != null) 1 else 0) + - (if (value4 != null) 1 else 0) + - (if (value5 != null) 1 else 0) + - (if (value6 != null) 1 else 0) - ) - - private fun MessagePacker.packKeyValue(name: String, value: Int?) { - if (value == null) return - packString(name) - packInt(value) - } - - private fun MessagePacker.packKeyValue(name: String, value: Long?) { - if (value == null) return - packString(name) - packLong(value) - } - - private fun MessagePacker.packKeyValue(name: String, value: String?) { - if (value == null) return - packString(name) - packString(value) - } - - private fun MessagePacker.packKeyValue(name: String, value: Collection?) { - if (value == null) return - packString(name) - packArrayHeader(value.size) - for (elem in value) packString(elem) - } - - private fun MessagePacker.packKeyValue(name: String, value: Map?) { - if (value == null) return - packString(name) - packMapHeader(value.size) - for ((k, v) in value) { - packString(k) - packString(v) - } - } - - private fun MessagePacker.packKeyValue(name: String, value: ByteArray?) { - if (value == null) return - packString(name) - packBinaryHeader(value.size) - writePayload(value) - } - - private fun MessagePacker.packKeyValue(name: String, value: Boolean) { - packString(name) - packBoolean(value) - } -} diff --git a/pkl-server/src/main/kotlin/org/pkl/server/MessageTransports.kt b/pkl-server/src/main/kotlin/org/pkl/server/MessageTransports.kt deleted file mode 100644 index 2b286ea69..000000000 --- a/pkl-server/src/main/kotlin/org/pkl/server/MessageTransports.kt +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.pkl.server - -import java.io.InputStream -import java.io.OutputStream -import java.util.concurrent.ConcurrentHashMap - -/** Factory methods for creating [MessageTransport]s. */ -object MessageTransports { - /** Creates a message transport that reads from [inputStream] and writes to [outputStream]. */ - fun stream(inputStream: InputStream, outputStream: OutputStream): MessageTransport { - return EncodingMessageTransport( - MessageDecoders.from(inputStream), - MessageEncoders.into(outputStream) - ) - } - - /** Creates "client" and "server" transports that are directly connected to each other. */ - fun direct(): Pair { - val transport1 = DirectMessageTransport() - val transport2 = DirectMessageTransport() - transport1.other = transport2 - transport2.other = transport1 - return transport1 to transport2 - } - - internal class EncodingMessageTransport( - private val decoder: MessageDecoder, - private val encoder: MessageEncoder, - ) : AbstractMessageTransport() { - @Volatile private var isClosed: Boolean = false - - override fun doStart() { - while (!isClosed) { - val message = decoder.decode() ?: return - accept(message) - } - } - - override fun doClose() { - isClosed = true - } - - override fun doSend(message: Message) { - encoder.encode(message) - } - } - - internal class DirectMessageTransport : AbstractMessageTransport() { - lateinit var other: DirectMessageTransport - - override fun doStart() {} - - override fun doClose() {} - - override fun doSend(message: Message) { - other.accept(message) - } - } - - // TODO: clean up callbacks if evaluation fails for some reason (ThreadInterrupt, timeout, etc) - internal abstract class AbstractMessageTransport : MessageTransport { - private lateinit var oneWayHandler: (OneWayMessage) -> Unit - private lateinit var requestHandler: (RequestMessage) -> Unit - private val responseHandlers: MutableMap Unit> = ConcurrentHashMap() - - protected abstract fun doStart() - - protected abstract fun doClose() - - protected abstract fun doSend(message: Message) - - protected fun accept(message: Message) { - log("Received message: $message") - when (message) { - is OneWayMessage -> oneWayHandler(message) - is RequestMessage -> requestHandler(message) - is ResponseMessage -> { - val handler = - responseHandlers.remove(message.requestId) - ?: throw ProtocolException( - "Received response ${message.javaClass.simpleName} for unknown request ID `${message.requestId}`." - ) - handler(message) - } - } - } - - final override fun start( - oneWayHandler: (OneWayMessage) -> Unit, - requestHandler: (RequestMessage) -> Unit - ) { - log("Starting transport: $this") - this.oneWayHandler = oneWayHandler - this.requestHandler = requestHandler - doStart() - } - - final override fun close() { - log("Closing transport: $this") - doClose() - responseHandlers.clear() - } - - override fun send(message: OneWayMessage) { - log("Sending message: $message") - doSend(message) - } - - override fun send(message: RequestMessage, responseHandler: (ResponseMessage) -> Unit) { - log("Sending message: $message") - responseHandlers[message.requestId] = responseHandler - return doSend(message) - } - - override fun send(message: ResponseMessage) { - log("Sending message: $message") - doSend(message) - } - } -} diff --git a/pkl-server/src/main/kotlin/org/pkl/server/Server.kt b/pkl-server/src/main/kotlin/org/pkl/server/Server.kt index 99df4ad9e..c8879bc6d 100644 --- a/pkl-server/src/main/kotlin/org/pkl/server/Server.kt +++ b/pkl-server/src/main/kotlin/org/pkl/server/Server.kt @@ -15,18 +15,27 @@ */ package org.pkl.server +import java.io.InputStream +import java.io.OutputStream import java.net.URI import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import kotlin.random.Random import org.pkl.core.* +import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader +import org.pkl.core.externalreader.ExternalReaderProcess +import org.pkl.core.externalreader.ExternalReaderProcessImpl import org.pkl.core.http.HttpClient +import org.pkl.core.messaging.MessageTransport +import org.pkl.core.messaging.MessageTransports +import org.pkl.core.messaging.ProtocolException import org.pkl.core.module.ModuleKeyFactories import org.pkl.core.module.ModuleKeyFactory import org.pkl.core.module.ModulePathResolver import org.pkl.core.packages.PackageUri import org.pkl.core.project.DeclaredDependencies +import org.pkl.core.resource.ExternalResourceResolver import org.pkl.core.resource.ResourceReader import org.pkl.core.resource.ResourceReaders import org.pkl.core.util.IoUtils @@ -37,6 +46,22 @@ class Server(private val transport: MessageTransport) : AutoCloseable { // https://github.com/jano7/executor would be the perfect executor here private val executor: ExecutorService = Executors.newSingleThreadExecutor() + // ExternalProcess instances with the same ExternalReader spec are shared per evaluator + private val externalReaderProcesses: + MutableMap> = + ConcurrentHashMap() + + companion object { + fun stream(inputStream: InputStream, outputStream: OutputStream): Server = + Server( + MessageTransports.stream( + ServerMessagePackDecoder(inputStream), + ServerMessagePackEncoder(outputStream), + ::log + ) + ) + } + /** Starts listening to incoming messages */ fun start() { transport.start( @@ -71,13 +96,13 @@ class Server(private val transport: MessageTransport) : AutoCloseable { private fun handleCreateEvaluator(message: CreateEvaluatorRequest) { val evaluatorId = Random.Default.nextLong() - val baseResponse = CreateEvaluatorResponse(message.requestId, evaluatorId = null, error = null) + val baseResponse = CreateEvaluatorResponse(message.requestId(), null, null) val evaluator = try { createEvaluator(message, evaluatorId) - } catch (e: ServerException) { - transport.send(baseResponse.copy(error = e.message)) + } catch (e: ProtocolException) { + transport.send(baseResponse.copy(error = e.message ?: "")) return } @@ -86,7 +111,7 @@ class Server(private val transport: MessageTransport) : AutoCloseable { } private fun handleEvaluate(msg: EvaluateRequest) { - val baseResponse = EvaluateResponse(msg.requestId, msg.evaluatorId, result = null, error = null) + val baseResponse = EvaluateResponse(msg.requestId(), msg.evaluatorId, null, null) val evaluator = evaluators[msg.evaluatorId] if (evaluator == null) { @@ -103,7 +128,7 @@ class Server(private val transport: MessageTransport) : AutoCloseable { } catch (e: PklBugException) { transport.send(baseResponse.copy(error = e.toString())) } catch (e: PklException) { - transport.send(baseResponse.copy(error = e.message)) + transport.send(baseResponse.copy(error = e.message ?: "")) } } } @@ -115,6 +140,9 @@ class Server(private val transport: MessageTransport) : AutoCloseable { return } evaluator.close() + + // close any running ExternalProcess instances for the closed evaluator + externalReaderProcesses[message.evaluatorId]?.values?.forEach { it.close() } } private fun buildDeclaredDependencies( @@ -167,8 +195,9 @@ class Server(private val transport: MessageTransport) : AutoCloseable { message.http?.proxy?.let { proxy -> setProxy(proxy.address, proxy.noProxy ?: listOf()) proxy.address?.let(IoUtils::setSystemProxy) + proxy.noProxy?.let { System.setProperty("http.nonProxyHosts", it.joinToString("|")) } } - message.http?.caCertificates?.let { caCertificates -> addCertificates(caCertificates) } + message.http?.caCertificates?.let(::addCertificates) buildLazily() } val dependencies = @@ -210,10 +239,19 @@ class Server(private val transport: MessageTransport) : AutoCloseable { add(ResourceReaders.pkg()) add(ResourceReaders.projectpackage()) add(ResourceReaders.modulePath(modulePathResolver)) + for ((scheme, spec) in message.externalResourceReaders ?: emptyMap()) { + add( + ResourceReaders.externalProcess(scheme, getExternalProcess(evaluatorId, spec), evaluatorId) + ) + } // add client-side resource readers last to ensure they win over builtin ones for (readerSpec in message.clientResourceReaders ?: emptyList()) { - val resourceReader = ClientResourceReader(transport, evaluatorId, readerSpec) - add(resourceReader) + add( + ResourceReaders.externalResolver( + readerSpec, + ExternalResourceResolver(transport, evaluatorId) + ) + ) } } @@ -226,6 +264,15 @@ class Server(private val transport: MessageTransport) : AutoCloseable { if (message.clientModuleReaders?.isNotEmpty() == true) { add(ClientModuleKeyFactory(message.clientModuleReaders, transport, evaluatorId)) } + for ((scheme, spec) in message.externalModuleReaders ?: emptyMap()) { + add( + ModuleKeyFactories.externalProcess( + scheme, + getExternalProcess(evaluatorId, spec), + evaluatorId + ) + ) + } add(ModuleKeyFactories.standardLibrary) addAll(ModuleKeyFactories.fromServiceProviders()) add(ModuleKeyFactories.file) @@ -235,4 +282,9 @@ class Server(private val transport: MessageTransport) : AutoCloseable { add(ModuleKeyFactories.http) add(ModuleKeyFactories.genericUrl) } + + private fun getExternalProcess(evaluatorId: Long, spec: ExternalReader): ExternalReaderProcess = + externalReaderProcesses + .computeIfAbsent(evaluatorId) { ConcurrentHashMap() } + .computeIfAbsent(spec) { ExternalReaderProcessImpl(it) } } diff --git a/pkl-server/src/main/kotlin/org/pkl/server/ServerMessagePackDecoder.kt b/pkl-server/src/main/kotlin/org/pkl/server/ServerMessagePackDecoder.kt new file mode 100644 index 000000000..ccfc78793 --- /dev/null +++ b/pkl-server/src/main/kotlin/org/pkl/server/ServerMessagePackDecoder.kt @@ -0,0 +1,134 @@ +/* + * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.server + +import java.io.InputStream +import java.net.URI +import java.nio.file.Path +import java.time.Duration +import java.util.regex.Pattern +import org.msgpack.core.MessagePack +import org.msgpack.core.MessageUnpacker +import org.msgpack.value.Value +import org.pkl.core.evaluatorSettings.PklEvaluatorSettings +import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader +import org.pkl.core.messaging.BaseMessagePackDecoder +import org.pkl.core.messaging.Message +import org.pkl.core.packages.Checksums + +class ServerMessagePackDecoder(unpacker: MessageUnpacker) : BaseMessagePackDecoder(unpacker) { + + constructor(stream: InputStream) : this(MessagePack.newDefaultUnpacker(stream)) + + override fun decodeMessage(msgType: Message.Type, map: Map): Message? { + return when (msgType) { + Message.Type.CREATE_EVALUATOR_REQUEST -> + CreateEvaluatorRequest( + get(map, "requestId").asIntegerValue().asLong(), + unpackStringListOrNull(map, "allowedModules", Pattern::compile), + unpackStringListOrNull(map, "allowedResources", Pattern::compile), + unpackListOrNull(map, "clientModuleReaders") { unpackModuleReaderSpec(it)!! }, + unpackListOrNull(map, "clientResourceReaders") { unpackResourceReaderSpec(it)!! }, + unpackStringListOrNull(map, "modulePaths", Path::of), + unpackStringMapOrNull(map, "env"), + unpackStringMapOrNull(map, "properties"), + unpackLongOrNull(map, "timeoutSeconds", Duration::ofSeconds), + unpackStringOrNull(map, "rootDir", Path::of), + unpackStringOrNull(map, "cacheDir", Path::of), + unpackStringOrNull(map, "outputFormat"), + map.unpackProject(), + map.unpackHttp(), + unpackStringMapOrNull(map, "externalModuleReaders", ::unpackExternalReader), + unpackStringMapOrNull(map, "externalResourceReaders", ::unpackExternalReader) + ) + Message.Type.CREATE_EVALUATOR_RESPONSE -> + CreateEvaluatorResponse( + unpackLong(map, "requestId"), + unpackLongOrNull(map, "evaluatorId"), + unpackStringOrNull(map, "error") + ) + Message.Type.CLOSE_EVALUATOR -> CloseEvaluator(unpackLong(map, "evaluatorId")) + Message.Type.EVALUATE_REQUEST -> + EvaluateRequest( + unpackLong(map, "requestId"), + unpackLong(map, "evaluatorId"), + URI(unpackString(map, "moduleUri")), + unpackStringOrNull(map, "moduleText"), + unpackStringOrNull(map, "expr") + ) + Message.Type.EVALUATE_RESPONSE -> + EvaluateResponse( + unpackLong(map, "requestId"), + unpackLong(map, "evaluatorId"), + unpackByteArray(map, "result"), + unpackStringOrNull(map, "error") + ) + Message.Type.LOG_MESSAGE -> + LogMessage( + unpackLong(map, "evaluatorId"), + unpackInt(map, "level"), + unpackString(map, "message"), + unpackString(map, "frameUri") + ) + else -> super.decodeMessage(msgType, map) + } + } + + private fun Map.unpackProject(): Project? { + val projMap = getNullable(this, "project")?.asMapValue()?.map() ?: return null + val projectFileUri = URI(unpackString(projMap, "projectFileUri")) + val dependencies = projMap.unpackDependencies("dependencies") + return Project(projectFileUri, null, dependencies) + } + + private fun Map.unpackHttp(): Http? { + val httpMap = getNullable(this, "http")?.asMapValue()?.map() ?: return null + val proxy = httpMap.unpackProxy() + val caCertificates = getNullable(httpMap, "caCertificates")?.asBinaryValue()?.asByteArray() + return Http(caCertificates, proxy) + } + + private fun Map.unpackProxy(): PklEvaluatorSettings.Proxy? { + val proxyMap = getNullable(this, "proxy")?.asMapValue()?.map() ?: return null + val address = unpackString(proxyMap, "address") + val noProxy = unpackStringListOrNull(proxyMap, "noProxy") + return PklEvaluatorSettings.Proxy.create(address, noProxy) + } + + private fun Map.unpackDependencies(name: String): Map { + val mapValue = get(this, name).asMapValue().map() + return mapValue.entries.associate { (key, value) -> + val dependencyName = key.asStringValue().asString() + val dependencyObj = value.asMapValue().map() + val type = unpackString(dependencyObj, "type") + val packageUri = URI(unpackString(dependencyObj, "packageUri")) + if (type == DependencyType.REMOTE.value) { + val checksums = + getNullable(dependencyObj, "checksums")?.asMapValue()?.map()?.let { obj -> + val sha256 = unpackString(obj, "sha256") + Checksums(sha256) + } + return@associate dependencyName to RemoteDependency(packageUri, checksums) + } + val dependencies = dependencyObj.unpackDependencies("dependencies") + val projectFileUri = unpackString(dependencyObj, "projectFileUri") + dependencyName to Project(URI(projectFileUri), packageUri, dependencies) + } + } + + private fun unpackExternalReader(map: Map): ExternalReader = + ExternalReader(unpackString(map, "executable"), unpackStringListOrNull(map, "arguments")!!) +} diff --git a/pkl-server/src/main/kotlin/org/pkl/server/ServerMessagePackEncoder.kt b/pkl-server/src/main/kotlin/org/pkl/server/ServerMessagePackEncoder.kt new file mode 100644 index 000000000..0c3193cff --- /dev/null +++ b/pkl-server/src/main/kotlin/org/pkl/server/ServerMessagePackEncoder.kt @@ -0,0 +1,197 @@ +/* + * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.server + +import java.io.OutputStream +import java.nio.file.Path +import kotlin.io.path.pathString +import org.msgpack.core.MessagePack +import org.msgpack.core.MessagePacker +import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader +import org.pkl.core.messaging.BaseMessagePackEncoder +import org.pkl.core.messaging.Message +import org.pkl.core.packages.Checksums + +class ServerMessagePackEncoder(packer: MessagePacker) : BaseMessagePackEncoder(packer) { + + constructor(stream: OutputStream) : this(MessagePack.newDefaultPacker(stream)) + + private fun MessagePacker.packProject(project: Project) { + packMapHeader(2) + packKeyValue("projectFileUri", project.projectFileUri.toString()) + packString("dependencies") + packDependencies(project.dependencies) + } + + private fun MessagePacker.packHttp(http: Http) { + packMapHeader(0, http.caCertificates, http.proxy) + http.caCertificates?.let { packKeyValue("caCertificates", it) } + http.proxy?.let { proxy -> + packString("proxy") + packMapHeader(0, proxy.address, proxy.noProxy) + packKeyValue("address", proxy.address?.toString()) + packKeyValue("noProxy", proxy.noProxy) + } + } + + private fun MessagePacker.packDependencies(dependencies: Map) { + packMapHeader(dependencies.size) + for ((name, dep) in dependencies) { + packString(name) + if (dep is Project) { + packMapHeader(4) + packKeyValue("type", dep.type.value) + packKeyValue("packageUri", dep.packageUri.toString()) + packKeyValue("projectFileUri", dep.projectFileUri.toString()) + packString("dependencies") + packDependencies(dep.dependencies) + } else { + dep as RemoteDependency + packMapHeader(dep.checksums?.let { 3 } ?: 2) + packKeyValue("type", dep.type.value) + packKeyValue("packageUri", dep.packageUri.toString()) + dep.checksums?.let { checksums -> + packString("checksums") + packChecksums(checksums) + } + } + } + } + + private fun MessagePacker.packChecksums(checksums: Checksums) { + packMapHeader(1) + packKeyValue("sha256", checksums.sha256) + } + + private fun MessagePacker.packExternalReader(spec: ExternalReader) { + packMapHeader(1, spec.arguments) + packKeyValue("executable", spec.executable) + spec.arguments?.let { packKeyValue("arguments", it) } + } + + override fun encodeMessage(msg: Message) { + when (msg.type()) { + Message.Type.CREATE_EVALUATOR_REQUEST -> { + msg as CreateEvaluatorRequest + packMapHeader( + 1, + msg.allowedModules, + msg.allowedResources, + msg.clientModuleReaders, + msg.clientResourceReaders, + msg.modulePaths, + msg.env, + msg.properties, + msg.timeout, + msg.rootDir, + msg.cacheDir, + msg.outputFormat, + msg.project, + msg.http, + msg.externalModuleReaders, + msg.externalResourceReaders, + ) + packKeyValue("requestId", msg.requestId()) + packKeyValue("allowedModules", msg.allowedModules?.map { it.toString() }) + packKeyValue("allowedResources", msg.allowedResources?.map { it.toString() }) + if (msg.clientModuleReaders != null) { + packer.packString("clientModuleReaders") + packer.packArrayHeader(msg.clientModuleReaders.size) + for (moduleReader in msg.clientModuleReaders) { + packModuleReaderSpec(moduleReader) + } + } + if (msg.clientResourceReaders != null) { + packer.packString("clientResourceReaders") + packer.packArrayHeader(msg.clientResourceReaders.size) + for (resourceReader in msg.clientResourceReaders) { + packResourceReaderSpec(resourceReader) + } + } + packKeyValue("modulePaths", msg.modulePaths, Path::toString) + packKeyValue("env", msg.env) + packKeyValue("properties", msg.properties) + packKeyValue("timeoutSeconds", msg.timeout?.toSeconds()) + packKeyValue("rootDir", msg.rootDir?.pathString) + packKeyValue("cacheDir", msg.cacheDir?.pathString) + packKeyValue("outputFormat", msg.outputFormat) + if (msg.project != null) { + packer.packString("project") + packer.packProject(msg.project) + } + if (msg.http != null) { + packer.packString("http") + packer.packHttp(msg.http) + } + if (msg.externalModuleReaders != null) { + packer.packString("externalModuleReaders") + packer.packMapHeader(msg.externalModuleReaders.size) + for ((scheme, spec) in msg.externalModuleReaders) { + packer.packString(scheme) + packer.packExternalReader(spec) + } + } + if (msg.externalResourceReaders != null) { + packer.packString("externalResourceReaders") + packer.packMapHeader(msg.externalResourceReaders.size) + for ((scheme, spec) in msg.externalResourceReaders) { + packer.packString(scheme) + packer.packExternalReader(spec) + } + } + return + } + Message.Type.CREATE_EVALUATOR_RESPONSE -> { + msg as CreateEvaluatorResponse + packMapHeader(1, msg.evaluatorId, msg.error) + packKeyValue("requestId", msg.requestId()) + packKeyValue("evaluatorId", msg.evaluatorId) + packKeyValue("error", msg.error) + } + Message.Type.CLOSE_EVALUATOR -> { + msg as CloseEvaluator + packer.packMapHeader(1) + packKeyValue("evaluatorId", msg.evaluatorId) + } + Message.Type.EVALUATE_REQUEST -> { + msg as EvaluateRequest + packMapHeader(3, msg.moduleText, msg.expr) + packKeyValue("requestId", msg.requestId()) + packKeyValue("evaluatorId", msg.evaluatorId) + packKeyValue("moduleUri", msg.moduleUri.toString()) + packKeyValue("moduleText", msg.moduleText) + packKeyValue("expr", msg.expr) + } + Message.Type.EVALUATE_RESPONSE -> { + msg as EvaluateResponse + packMapHeader(2, msg.result, msg.error) + packKeyValue("requestId", msg.requestId()) + packKeyValue("evaluatorId", msg.evaluatorId) + msg.result?.let { packKeyValue("result", it) } + packKeyValue("error", msg.error) + } + Message.Type.LOG_MESSAGE -> { + msg as LogMessage + packer.packMapHeader(4) + packKeyValue("evaluatorId", msg.evaluatorId) + packKeyValue("level", msg.level) + packKeyValue("message", msg.message) + packKeyValue("frameUri", msg.frameUri) + } + else -> super.encodeMessage(msg) + } + } +} diff --git a/pkl-server/src/main/kotlin/org/pkl/server/Message.kt b/pkl-server/src/main/kotlin/org/pkl/server/ServerMessages.kt similarity index 54% rename from pkl-server/src/main/kotlin/org/pkl/server/Message.kt rename to pkl-server/src/main/kotlin/org/pkl/server/ServerMessages.kt index fe7fd3edf..1e7562d67 100644 --- a/pkl-server/src/main/kotlin/org/pkl/server/Message.kt +++ b/pkl-server/src/main/kotlin/org/pkl/server/ServerMessages.kt @@ -20,98 +20,18 @@ import java.nio.file.Path import java.time.Duration import java.util.* import java.util.regex.Pattern +import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.Proxy -import org.pkl.core.module.PathElement +import org.pkl.core.messaging.Message +import org.pkl.core.messaging.Messages.* import org.pkl.core.packages.Checksums -sealed interface Message { - val type: MessageType -} - -sealed interface OneWayMessage : Message - -sealed interface RequestMessage : Message { - val requestId: Long -} - -sealed interface ResponseMessage : Message { - val requestId: Long -} - -sealed class ClientMessage : Message - -sealed class ClientRequestMessage : ClientMessage(), RequestMessage - -sealed class ClientResponseMessage : ClientMessage(), ResponseMessage - -sealed class ClientOneWayMessage : ClientMessage(), OneWayMessage - -sealed class ServerMessage : Message - -sealed class ServerRequestMessage : ServerMessage(), RequestMessage - -sealed class ServerResponseMessage : ServerMessage(), ResponseMessage - -sealed class ServerOneWayMessage : ServerMessage(), OneWayMessage - -enum class MessageType(val code: Int) { - CREATE_EVALUATOR_REQUEST(0x20), - CREATE_EVALUATOR_RESPONSE(0x21), - CLOSE_EVALUATOR(0x22), - EVALUATE_REQUEST(0x23), - EVALUATE_RESPONSE(0x24), - LOG_MESSAGE(0x25), - READ_RESOURCE_REQUEST(0x26), - READ_RESOURCE_RESPONSE(0x27), - READ_MODULE_REQUEST(0x28), - READ_MODULE_RESPONSE(0x29), - LIST_RESOURCES_REQUEST(0x2a), - LIST_RESOURCES_RESPONSE(0x2b), - LIST_MODULES_REQUEST(0x2c), - LIST_MODULES_RESPONSE(0x2d), -} - -data class ModuleReaderSpec( - val scheme: String, - val hasHierarchicalUris: Boolean, - val isLocal: Boolean, - val isGlobbable: Boolean -) - -data class ResourceReaderSpec( - val scheme: String, - val hasHierarchicalUris: Boolean, - val isGlobbable: Boolean, -) - private fun T?.equalsNullable(other: Any?): Boolean { return Objects.equals(this, other) } -enum class DependencyType(val value: String) { - LOCAL("local"), - REMOTE("remote") -} - -sealed interface Dependency { - val type: DependencyType - val packageUri: URI? -} - -data class RemoteDependency(override val packageUri: URI, val checksums: Checksums?) : Dependency { - override val type: DependencyType = DependencyType.REMOTE -} - -data class Project( - val projectFileUri: URI, - override val packageUri: URI?, - val dependencies: Map -) : Dependency { - override val type: DependencyType = DependencyType.LOCAL -} - data class CreateEvaluatorRequest( - override val requestId: Long, + private val requestId: Long, val allowedModules: List?, val allowedResources: List?, val clientModuleReaders: List?, @@ -124,9 +44,14 @@ data class CreateEvaluatorRequest( val cacheDir: Path?, val outputFormat: String?, val project: Project?, - val http: Http? -) : ClientRequestMessage() { - override val type = MessageType.CREATE_EVALUATOR_REQUEST + val http: Http?, + val externalModuleReaders: Map?, + val externalResourceReaders: Map? +) : Message.Client.Request { + + override fun type(): Message.Type = Message.Type.CREATE_EVALUATOR_REQUEST + + override fun requestId(): Long = requestId // need to implement this manually because [Pattern.equals] returns false for two patterns // that have the same underlying pattern string. @@ -152,7 +77,9 @@ data class CreateEvaluatorRequest( cacheDir.equalsNullable(other.cacheDir) && outputFormat.equalsNullable(other.outputFormat) && project.equalsNullable(other.project) && - http.equalsNullable(other.http) + http.equalsNullable(other.http) && + externalModuleReaders.equalsNullable(other.externalModuleReaders) && + externalResourceReaders.equalsNullable(other.externalResourceReaders) } @Suppress("DuplicatedCode") // false duplicate within method @@ -170,8 +97,9 @@ data class CreateEvaluatorRequest( result = 31 * result + cacheDir.hashCode() result = 31 * result + outputFormat.hashCode() result = 31 * result + project.hashCode() - result = 31 * result + type.hashCode() result = 31 * result + http.hashCode() + result = 31 * result + externalModuleReaders.hashCode() + result = 31 * result + externalResourceReaders.hashCode() return result } } @@ -200,69 +128,63 @@ data class Http( } } -data class CreateEvaluatorResponse( - override val requestId: Long, - val evaluatorId: Long?, - val error: String?, -) : ServerResponseMessage() { - override val type - get() = MessageType.CREATE_EVALUATOR_RESPONSE +enum class DependencyType(val value: String) { + LOCAL("local"), + REMOTE("remote") } -data class ListResourcesRequest(override val requestId: Long, val evaluatorId: Long, val uri: URI) : - ServerRequestMessage() { - override val type: MessageType - get() = MessageType.LIST_RESOURCES_REQUEST +sealed interface Dependency { + val type: DependencyType + val packageUri: URI? } -data class ListResourcesResponse( - override val requestId: Long, - val evaluatorId: Long, - val pathElements: List?, - val error: String? -) : ClientResponseMessage() { - override val type: MessageType - get() = MessageType.LIST_RESOURCES_RESPONSE +data class RemoteDependency(override val packageUri: URI, val checksums: Checksums?) : Dependency { + override val type: DependencyType = DependencyType.REMOTE } -data class ListModulesRequest(override val requestId: Long, val evaluatorId: Long, val uri: URI) : - ServerRequestMessage() { - override val type: MessageType - get() = MessageType.LIST_MODULES_REQUEST +data class Project( + val projectFileUri: URI, + override val packageUri: URI?, + val dependencies: Map +) : Dependency { + override val type: DependencyType = DependencyType.LOCAL } -data class ListModulesResponse( - override val requestId: Long, - val evaluatorId: Long, - val pathElements: List?, - val error: String? -) : ClientResponseMessage() { - override val type: MessageType - get() = MessageType.LIST_MODULES_RESPONSE +data class CreateEvaluatorResponse( + private val requestId: Long, + val evaluatorId: Long?, + val error: String?, +) : Message.Server.Response { + override fun type(): Message.Type = Message.Type.CREATE_EVALUATOR_RESPONSE + + override fun requestId(): Long = requestId } -data class CloseEvaluator(val evaluatorId: Long) : ClientOneWayMessage() { - override val type = MessageType.CLOSE_EVALUATOR +data class CloseEvaluator(val evaluatorId: Long) : Message.Client.OneWay { + override fun type(): Message.Type = Message.Type.CLOSE_EVALUATOR } data class EvaluateRequest( - override val requestId: Long, + private val requestId: Long, val evaluatorId: Long, val moduleUri: URI, val moduleText: String?, val expr: String? -) : ClientRequestMessage() { - override val type = MessageType.EVALUATE_REQUEST +) : Message.Client.Request { + override fun type(): Message.Type = Message.Type.EVALUATE_REQUEST + + override fun requestId(): Long = requestId } data class EvaluateResponse( - override val requestId: Long, + private val requestId: Long, val evaluatorId: Long, val result: ByteArray?, val error: String? -) : ServerResponseMessage() { - override val type - get() = MessageType.EVALUATE_RESPONSE +) : Message.Server.Response { + override fun type(): Message.Type = Message.Type.EVALUATE_RESPONSE + + override fun requestId(): Long = requestId // override to use [ByteArray.contentEquals] @Suppress("DuplicatedCode") @@ -291,58 +213,6 @@ data class LogMessage( val level: Int, val message: String, val frameUri: String -) : ServerOneWayMessage() { - override val type - get() = MessageType.LOG_MESSAGE -} - -data class ReadResourceRequest(override val requestId: Long, val evaluatorId: Long, val uri: URI) : - ServerRequestMessage() { - override val type - get() = MessageType.READ_RESOURCE_REQUEST -} - -data class ReadResourceResponse( - override val requestId: Long, - val evaluatorId: Long, - val contents: ByteArray?, - val error: String? -) : ClientResponseMessage() { - override val type = MessageType.READ_RESOURCE_RESPONSE - - // override to use [ByteArray.contentEquals] - @Suppress("DuplicatedCode") - override fun equals(other: Any?): Boolean { - if (this === other) return true - if (other !is ReadResourceResponse) return false - - return requestId == other.requestId && - evaluatorId == other.evaluatorId && - contents.contentEquals(other.contents) && - error == other.error - } - - // override to use [ByteArray.contentHashCode] - override fun hashCode(): Int { - var result = requestId.hashCode() - result = 31 * result + evaluatorId.hashCode() - result = 31 * result + contents.contentHashCode() - result = 31 * result + error.hashCode() - return result - } -} - -data class ReadModuleRequest(override val requestId: Long, val evaluatorId: Long, val uri: URI) : - ServerRequestMessage() { - override val type - get() = MessageType.READ_MODULE_REQUEST -} - -data class ReadModuleResponse( - override val requestId: Long, - val evaluatorId: Long, - val contents: String?, - val error: String? -) : ClientResponseMessage() { - override val type = MessageType.READ_MODULE_RESPONSE +) : Message.Server.OneWay { + override fun type(): Message.Type = Message.Type.LOG_MESSAGE } diff --git a/pkl-server/src/main/kotlin/org/pkl/server/Utils.kt b/pkl-server/src/main/kotlin/org/pkl/server/Utils.kt index 72280160c..c5b7536d3 100644 --- a/pkl-server/src/main/kotlin/org/pkl/server/Utils.kt +++ b/pkl-server/src/main/kotlin/org/pkl/server/Utils.kt @@ -20,6 +20,7 @@ import java.util.concurrent.ExecutionException import java.util.concurrent.Future import org.msgpack.core.MessageBufferPacker import org.msgpack.core.MessagePack +import org.pkl.core.messaging.Message internal fun log(msg: String) { if (System.getenv("PKL_DEBUG") == "1") { @@ -41,7 +42,7 @@ internal val threadLocalBufferPacker: ThreadLocal = private val threadLocalEncoder: ThreadLocal<(Message) -> ByteArray> = ThreadLocal.withInitial { val packer = threadLocalBufferPacker.get() - val encoder = MessageEncoders.into(packer); + val encoder = ServerMessagePackEncoder(packer); { message: Message -> packer.clear() encoder.encode(message) diff --git a/pkl-server/src/test/kotlin/org/pkl/server/AbstractServerTest.kt b/pkl-server/src/test/kotlin/org/pkl/server/AbstractServerTest.kt index 699196fd1..d18c58f9b 100644 --- a/pkl-server/src/test/kotlin/org/pkl/server/AbstractServerTest.kt +++ b/pkl-server/src/test/kotlin/org/pkl/server/AbstractServerTest.kt @@ -25,10 +25,12 @@ import kotlin.io.path.outputStream import kotlin.io.path.writeText import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.AfterAll +import org.junit.jupiter.api.Disabled import org.junit.jupiter.api.Test import org.junit.jupiter.api.io.TempDir import org.msgpack.core.MessagePack import org.pkl.commons.test.PackageServer +import org.pkl.core.messaging.Messages.* import org.pkl.core.module.PathElement abstract class AbstractServerTest { @@ -55,8 +57,8 @@ abstract class AbstractServerTest { @Test fun `create and close evaluator`() { - val evaluatorId = client.sendCreateEvaluatorRequest(requestId = 123) - client.send(CloseEvaluator(evaluatorId = evaluatorId)) + val evaluatorId = client.sendCreateEvaluatorRequest(123) + client.send(CloseEvaluator(evaluatorId)) } @Test @@ -66,24 +68,23 @@ abstract class AbstractServerTest { client.send( EvaluateRequest( - requestId = requestId, - evaluatorId = evaluatorId, - moduleUri = URI("repl:text"), - moduleText = - """ + requestId, + evaluatorId, + URI("repl:text"), + """ foo { bar = "bar" } """ - .trimIndent(), - expr = null + .trimIndent(), + null ) ) val response = client.receive() assertThat(response.error).isNull() assertThat(response.result).isNotNull - assertThat(response.requestId).isEqualTo(requestId) + assertThat(response.requestId()).isEqualTo(requestId) val unpacker = MessagePack.newDefaultUnpacker(response.result) val value = unpacker.unpackValue() @@ -96,15 +97,14 @@ abstract class AbstractServerTest { client.send( EvaluateRequest( - requestId = 1, - evaluatorId = evaluatorId, - moduleUri = URI("repl:text"), - moduleText = - """ + 1, + evaluatorId, + URI("repl:text"), + """ foo = trace(1 + 2 + 3) """ - .trimIndent(), - expr = null + .trimIndent(), + null ) ) @@ -121,18 +121,17 @@ abstract class AbstractServerTest { client.send( EvaluateRequest( - requestId = 1, - evaluatorId = evaluatorId, - moduleUri = URI("repl:text"), - moduleText = - """ + 1, + evaluatorId, + URI("repl:text"), + """ @Deprecated { message = "use bar instead" } function foo() = 5 result = foo() """ - .trimIndent(), - expr = null + .trimIndent(), + null ) ) @@ -145,17 +144,16 @@ abstract class AbstractServerTest { @Test fun `read resource`() { - val reader = - ResourceReaderSpec(scheme = "bahumbug", hasHierarchicalUris = true, isGlobbable = false) + val reader = ResourceReaderSpec("bahumbug", true, false) val evaluatorId = client.sendCreateEvaluatorRequest(resourceReaders = listOf(reader)) client.send( EvaluateRequest( - requestId = 1, - evaluatorId = evaluatorId, - moduleUri = URI("repl:text"), - moduleText = """res = read("bahumbug:/foo.pkl").text""", - expr = "res" + 1, + evaluatorId, + URI("repl:text"), + """res = read("bahumbug:/foo.pkl").text""", + "res" ) ) @@ -165,10 +163,10 @@ abstract class AbstractServerTest { client.send( ReadResourceResponse( - requestId = readResourceMsg.requestId, - evaluatorId = evaluatorId, - contents = "my bahumbug".toByteArray(), - error = null + readResourceMsg.requestId, + evaluatorId, + "my bahumbug".toByteArray(), + null ) ) @@ -180,10 +178,12 @@ abstract class AbstractServerTest { assertThat(value.asStringValue().asString()).isEqualTo("my bahumbug") } + @Disabled( + "Unable to construct ReadResourceResponse with null contents due to Kotlin compiler bug" + ) @Test fun `read resource -- null contents and null error`() { - val reader = - ResourceReaderSpec(scheme = "bahumbug", hasHierarchicalUris = true, isGlobbable = false) + val reader = ResourceReaderSpec("bahumbug", true, false) val evaluatorId = client.sendCreateEvaluatorRequest(resourceReaders = listOf(reader)) client.send( @@ -200,14 +200,11 @@ abstract class AbstractServerTest { assertThat(readResourceMsg.uri.toString()).isEqualTo("bahumbug:/foo.pkl") assertThat(readResourceMsg.evaluatorId).isEqualTo(evaluatorId) - client.send( - ReadResourceResponse( - requestId = readResourceMsg.requestId, - evaluatorId = evaluatorId, - contents = null, - error = null - ) - ) + client.send(ReadResourceResponse(readResourceMsg.requestId, evaluatorId, byteArrayOf(), null)) + // for this test to be correct this should actually be: + // client.send(ReadResourceResponse(readResourceMsg.requestId, evaluatorId, null, null)) + // this should be evaluated again once https://github.com/apple/pkl/issues/698 is addressed + // see conversation here https://github.com/apple/pkl/pull/660#discussion_r1819545811 val evaluateResponse = client.receive() assertThat(evaluateResponse.error).isNull() @@ -219,17 +216,16 @@ abstract class AbstractServerTest { @Test fun `read resource error`() { - val reader = - ResourceReaderSpec(scheme = "bahumbug", hasHierarchicalUris = true, isGlobbable = false) + val reader = ResourceReaderSpec("bahumbug", true, false) val evaluatorId = client.sendCreateEvaluatorRequest(resourceReaders = listOf(reader)) client.send( EvaluateRequest( - requestId = 1, - evaluatorId = evaluatorId, - moduleUri = URI("repl:text"), - moduleText = """res = read("bahumbug:/foo.txt").text""", - expr = "res" + 1, + evaluatorId, + URI("repl:text"), + """res = read("bahumbug:/foo.txt").text""", + "res" ) ) @@ -237,10 +233,10 @@ abstract class AbstractServerTest { client.send( ReadResourceResponse( - requestId = readResourceMsg.requestId, - evaluatorId = evaluatorId, - contents = null, - error = "cannot read my bahumbug" + readResourceMsg.requestId, + evaluatorId, + byteArrayOf(), + "cannot read my bahumbug" ) ) @@ -251,46 +247,44 @@ abstract class AbstractServerTest { @Test fun `glob resource`() { - val reader = ResourceReaderSpec(scheme = "bird", hasHierarchicalUris = true, isGlobbable = true) + val reader = ResourceReaderSpec("bird", true, true) val evaluatorId = client.sendCreateEvaluatorRequest(resourceReaders = listOf(reader)) client.send( EvaluateRequest( - requestId = 1, - evaluatorId = evaluatorId, - moduleUri = URI("repl:text"), - moduleText = - """ + 1, + evaluatorId, + URI("repl:text"), + """ res = read*("bird:/**.txt").keys """ - .trimIndent(), - expr = "res" + .trimIndent(), + "res" ) ) val listResourcesRequest = client.receive() assertThat(listResourcesRequest.uri.toString()).isEqualTo("bird:/") client.send( ListResourcesResponse( - requestId = listResourcesRequest.requestId, - evaluatorId = listResourcesRequest.evaluatorId, - pathElements = listOf(PathElement("foo.txt", false), PathElement("subdir", true)), - error = null + listResourcesRequest.requestId, + listResourcesRequest.evaluatorId, + listOf(PathElement("foo.txt", false), PathElement("subdir", true)), + null ) ) val listResourcesRequest2 = client.receive() assertThat(listResourcesRequest2.uri.toString()).isEqualTo("bird:/subdir/") client.send( ListResourcesResponse( - requestId = listResourcesRequest2.requestId, - evaluatorId = listResourcesRequest2.evaluatorId, - pathElements = - listOf( - PathElement("bar.txt", false), - ), - error = null + listResourcesRequest2.requestId, + listResourcesRequest2.evaluatorId, + listOf( + PathElement("bar.txt", false), + ), + null ) ) val evaluateResponse = client.receive() - assertThat(evaluateResponse.result!!.debugYaml) + assertThat(evaluateResponse.result?.debugYaml) .isEqualTo( """ - 6 @@ -304,32 +298,31 @@ abstract class AbstractServerTest { @Test fun `glob resources -- null pathElements and null error`() { - val reader = ResourceReaderSpec(scheme = "bird", hasHierarchicalUris = true, isGlobbable = true) + val reader = ResourceReaderSpec("bird", true, true) val evaluatorId = client.sendCreateEvaluatorRequest(resourceReaders = listOf(reader)) client.send( EvaluateRequest( - requestId = 1, - evaluatorId = evaluatorId, - moduleUri = URI("repl:text"), - moduleText = - """ + 1, + evaluatorId, + URI("repl:text"), + """ res = read*("bird:/**.txt").keys """ - .trimIndent(), - expr = "res" + .trimIndent(), + "res" ) ) val listResourcesRequest = client.receive() client.send( ListResourcesResponse( - requestId = listResourcesRequest.requestId, - evaluatorId = listResourcesRequest.evaluatorId, - pathElements = null, - error = null + listResourcesRequest.requestId, + listResourcesRequest.evaluatorId, + null, + null ) ) val evaluateResponse = client.receive() - assertThat(evaluateResponse.result!!.debugYaml) + assertThat(evaluateResponse.result?.debugYaml) .isEqualTo( """ - 6 @@ -341,29 +334,28 @@ abstract class AbstractServerTest { @Test fun `glob resource error`() { - val reader = ResourceReaderSpec(scheme = "bird", hasHierarchicalUris = true, isGlobbable = true) + val reader = ResourceReaderSpec("bird", true, true) val evaluatorId = client.sendCreateEvaluatorRequest(resourceReaders = listOf(reader)) client.send( EvaluateRequest( - requestId = 1, - evaluatorId = evaluatorId, - moduleUri = URI("repl:text"), - moduleText = - """ + 1, + evaluatorId, + URI("repl:text"), + """ res = read*("bird:/**.txt").keys """ - .trimIndent(), - expr = "res" + .trimIndent(), + "res" ) ) val listResourcesRequest = client.receive() assertThat(listResourcesRequest.uri.toString()).isEqualTo("bird:/") client.send( ListResourcesResponse( - requestId = listResourcesRequest.requestId, - evaluatorId = listResourcesRequest.evaluatorId, - pathElements = null, - error = "didnt work" + listResourcesRequest.requestId, + listResourcesRequest.evaluatorId, + null, + "didnt work" ) ) val evaluateResponse = client.receive() @@ -389,22 +381,16 @@ abstract class AbstractServerTest { @Test fun `read module`() { - val reader = - ModuleReaderSpec( - scheme = "bird", - hasHierarchicalUris = true, - isLocal = true, - isGlobbable = false - ) + val reader = ModuleReaderSpec("bird", true, true, false) val evaluatorId = client.sendCreateEvaluatorRequest(moduleReaders = listOf(reader)) client.send( EvaluateRequest( - requestId = 1, - evaluatorId = evaluatorId, - moduleUri = URI("repl:text"), - moduleText = """res = import("bird:/pigeon.pkl").value""", - expr = "res" + 1, + evaluatorId, + URI("repl:text"), + """res = import("bird:/pigeon.pkl").value""", + "res" ) ) @@ -412,14 +398,7 @@ abstract class AbstractServerTest { assertThat(readModuleMsg.uri.toString()).isEqualTo("bird:/pigeon.pkl") assertThat(readModuleMsg.evaluatorId).isEqualTo(evaluatorId) - client.send( - ReadModuleResponse( - requestId = readModuleMsg.requestId, - evaluatorId = evaluatorId, - contents = "value = 5", - error = null - ) - ) + client.send(ReadModuleResponse(readModuleMsg.requestId, evaluatorId, "value = 5", null)) val evaluateResponse = client.receive() assertThat(evaluateResponse.error).isNull() @@ -430,13 +409,7 @@ abstract class AbstractServerTest { @Test fun `read module -- null contents and null error`() { - val reader = - ModuleReaderSpec( - scheme = "bird", - hasHierarchicalUris = true, - isLocal = true, - isGlobbable = false - ) + val reader = ModuleReaderSpec("bird", true, true, false) val evaluatorId = client.sendCreateEvaluatorRequest(moduleReaders = listOf(reader)) client.send( @@ -453,14 +426,7 @@ abstract class AbstractServerTest { assertThat(readModuleMsg.uri.toString()).isEqualTo("bird:/pigeon.pkl") assertThat(readModuleMsg.evaluatorId).isEqualTo(evaluatorId) - client.send( - ReadModuleResponse( - requestId = readModuleMsg.requestId, - evaluatorId = evaluatorId, - contents = null, - error = null - ) - ) + client.send(ReadModuleResponse(readModuleMsg.requestId, evaluatorId, null, null)) val evaluateResponse = client.receive() assertThat(evaluateResponse.error).isNull() @@ -473,22 +439,16 @@ abstract class AbstractServerTest { @Test fun `read module error`() { - val reader = - ModuleReaderSpec( - scheme = "bird", - hasHierarchicalUris = true, - isLocal = true, - isGlobbable = false - ) + val reader = ModuleReaderSpec("bird", true, true, false) val evaluatorId = client.sendCreateEvaluatorRequest(moduleReaders = listOf(reader)) client.send( EvaluateRequest( - requestId = 1, - evaluatorId = evaluatorId, - moduleUri = URI("repl:text"), - moduleText = """res = import("bird:/pigeon.pkl").value""", - expr = "res" + 1, + evaluatorId, + URI("repl:text"), + """res = import("bird:/pigeon.pkl").value""", + "res" ) ) @@ -497,12 +457,7 @@ abstract class AbstractServerTest { assertThat(readModuleMsg.evaluatorId).isEqualTo(evaluatorId) client.send( - ReadModuleResponse( - requestId = readModuleMsg.requestId, - evaluatorId = evaluatorId, - contents = null, - error = "Don't know where Pigeon is" - ) + ReadModuleResponse(readModuleMsg.requestId, evaluatorId, null, "Don't know where Pigeon is") ) val evaluateResponse = client.receive() @@ -511,22 +466,16 @@ abstract class AbstractServerTest { @Test fun `glob module`() { - val reader = - ModuleReaderSpec( - scheme = "bird", - hasHierarchicalUris = true, - isLocal = true, - isGlobbable = true - ) + val reader = ModuleReaderSpec("bird", true, true, true) val evaluatorId = client.sendCreateEvaluatorRequest(moduleReaders = listOf(reader)) client.send( EvaluateRequest( - requestId = 1, - evaluatorId = evaluatorId, - moduleUri = URI("repl:text"), - moduleText = """res = import*("bird:/**.pkl").keys""", - expr = "res" + 1, + evaluatorId, + URI("repl:text"), + """res = import*("bird:/**.pkl").keys""", + "res" ) ) @@ -535,15 +484,14 @@ abstract class AbstractServerTest { assertThat(listModulesMsg.uri.path).isEqualTo("/") client.send( ListModulesResponse( - requestId = listModulesMsg.requestId, - evaluatorId = evaluatorId, - pathElements = - listOf( - PathElement("birds", true), - PathElement("majesticBirds", true), - PathElement("Person.pkl", false) - ), - error = null + listModulesMsg.requestId, + evaluatorId, + listOf( + PathElement("birds", true), + PathElement("majesticBirds", true), + PathElement("Person.pkl", false) + ), + null ) ) val listModulesMsg2 = client.receive() @@ -551,14 +499,13 @@ abstract class AbstractServerTest { assertThat(listModulesMsg2.uri.path).isEqualTo("/birds/") client.send( ListModulesResponse( - requestId = listModulesMsg2.requestId, - evaluatorId = listModulesMsg2.evaluatorId, - pathElements = - listOf( - PathElement("pigeon.pkl", false), - PathElement("parrot.pkl", false), - ), - error = null + listModulesMsg2.requestId, + listModulesMsg2.evaluatorId, + listOf( + PathElement("pigeon.pkl", false), + PathElement("parrot.pkl", false), + ), + null ) ) val listModulesMsg3 = client.receive() @@ -566,19 +513,18 @@ abstract class AbstractServerTest { assertThat(listModulesMsg3.uri.path).isEqualTo("/majesticBirds/") client.send( ListModulesResponse( - requestId = listModulesMsg3.requestId, - evaluatorId = listModulesMsg3.evaluatorId, - pathElements = - listOf( - PathElement("barnOwl.pkl", false), - PathElement("elfOwl.pkl", false), - ), - error = null + listModulesMsg3.requestId, + listModulesMsg3.evaluatorId, + listOf( + PathElement("barnOwl.pkl", false), + PathElement("elfOwl.pkl", false), + ), + null ) ) val evaluateResponse = client.receive() - assertThat(evaluateResponse.result!!.debugRendering) + assertThat(evaluateResponse.result?.debugRendering) .isEqualTo( """ - 6 @@ -595,36 +541,23 @@ abstract class AbstractServerTest { @Test fun `glob module -- null pathElements and null error`() { - val reader = - ModuleReaderSpec( - scheme = "bird", - hasHierarchicalUris = true, - isLocal = true, - isGlobbable = true - ) + val reader = ModuleReaderSpec("bird", true, true, true) val evaluatorId = client.sendCreateEvaluatorRequest(moduleReaders = listOf(reader)) client.send( EvaluateRequest( - requestId = 1, - evaluatorId = evaluatorId, - moduleUri = URI("repl:text"), - moduleText = """res = import*("bird:/**.pkl").keys""", - expr = "res" + 1, + evaluatorId, + URI("repl:text"), + """res = import*("bird:/**.pkl").keys""", + "res" ) ) val listModulesMsg = client.receive() - client.send( - ListModulesResponse( - requestId = listModulesMsg.requestId, - evaluatorId = evaluatorId, - pathElements = null, - error = null - ) - ) + client.send(ListModulesResponse(listModulesMsg.requestId, evaluatorId, null, null)) val evaluateResponse = client.receive() - assertThat(evaluateResponse.result!!.debugRendering) + assertThat(evaluateResponse.result?.debugRendering) .isEqualTo( """ - 6 @@ -636,36 +569,23 @@ abstract class AbstractServerTest { @Test fun `glob module error`() { - val reader = - ModuleReaderSpec( - scheme = "bird", - hasHierarchicalUris = true, - isLocal = true, - isGlobbable = true - ) + val reader = ModuleReaderSpec("bird", true, true, true) val evaluatorId = client.sendCreateEvaluatorRequest(moduleReaders = listOf(reader)) client.send( EvaluateRequest( - requestId = 1, - evaluatorId = evaluatorId, - moduleUri = URI("repl:text"), - moduleText = """res = import*("bird:/**.pkl").keys""", - expr = "res" + 1, + evaluatorId, + URI("repl:text"), + """res = import*("bird:/**.pkl").keys""", + "res" ) ) val listModulesMsg = client.receive() assertThat(listModulesMsg.uri.scheme).isEqualTo("bird") assertThat(listModulesMsg.uri.path).isEqualTo("/") - client.send( - ListModulesResponse( - requestId = listModulesMsg.requestId, - evaluatorId = evaluatorId, - pathElements = null, - error = "nope" - ) - ) + client.send(ListModulesResponse(listModulesMsg.requestId, evaluatorId, null, "nope")) val evaluateResponse = client.receive() assertThat(evaluateResponse.error) .isEqualTo( @@ -699,19 +619,13 @@ abstract class AbstractServerTest { val evaluatorId = client.sendCreateEvaluatorRequest(modulePaths = listOf(jarFile)) client.send( - EvaluateRequest( - requestId = 1, - evaluatorId = evaluatorId, - moduleUri = URI("modulepath:/dir1/module.pkl"), - moduleText = null, - expr = "output.text" - ) + EvaluateRequest(1, evaluatorId, URI("modulepath:/dir1/module.pkl"), null, "output.text") ) val response = client.receive() assertThat(response.error).isNull() val tripleQuote = "\"\"\"" - assertThat(response.result!!.debugYaml) + assertThat(response.result?.debugYaml) .isEqualTo( """ | @@ -741,38 +655,31 @@ abstract class AbstractServerTest { @Test fun `import triple-dot path`() { - val reader = - ModuleReaderSpec( - scheme = "bird", - hasHierarchicalUris = true, - isLocal = true, - isGlobbable = true - ) + val reader = ModuleReaderSpec("bird", true, true, true) val evaluatorId = client.sendCreateEvaluatorRequest(moduleReaders = listOf(reader)) client.send( EvaluateRequest( - requestId = 1, - evaluatorId = evaluatorId, - moduleUri = URI("bird:/foo/bar/baz.pkl"), - moduleText = - """ + 1, + evaluatorId, + URI("bird:/foo/bar/baz.pkl"), + """ import ".../buz.pkl" res = buz.res """ - .trimIndent(), - expr = "res" + .trimIndent(), + "res" ) ) val readModuleRequest = client.receive() assertThat(readModuleRequest.uri).isEqualTo(URI("bird:/foo/buz.pkl")) client.send( ReadModuleResponse( - requestId = readModuleRequest.requestId, - evaluatorId = readModuleRequest.evaluatorId, - contents = null, - error = "not here" + readModuleRequest.requestId, + readModuleRequest.evaluatorId, + null, + "not here" ) ) @@ -780,54 +687,40 @@ abstract class AbstractServerTest { assertThat(readModuleRequest2.uri).isEqualTo(URI("bird:/buz.pkl")) client.send( ReadModuleResponse( - requestId = readModuleRequest2.requestId, - evaluatorId = readModuleRequest2.evaluatorId, - contents = "res = 1", - error = null + readModuleRequest2.requestId, + readModuleRequest2.evaluatorId, + "res = 1", + null ) ) val evaluatorResponse = client.receive() - assertThat(evaluatorResponse.result!!.debugYaml).isEqualTo("1") + assertThat(evaluatorResponse.result?.debugYaml).isEqualTo("1") } @Test fun `evaluate error`() { val evaluatorId = client.sendCreateEvaluatorRequest() - client.send( - EvaluateRequest( - requestId = 1, - evaluatorId = evaluatorId, - moduleUri = URI("repl:text"), - moduleText = """foo = 1""", - expr = "foo as String" - ) - ) + client.send(EvaluateRequest(1, evaluatorId, URI("repl:text"), """foo = 1""", "foo as String")) val evaluateResponse = client.receive() - assertThat(evaluateResponse.requestId).isEqualTo(1) + assertThat(evaluateResponse.requestId()).isEqualTo(1) assertThat(evaluateResponse.error).contains("Expected value of type") } @Test fun `evaluate client-provided module reader`() { - val reader = - ModuleReaderSpec( - scheme = "bird", - hasHierarchicalUris = true, - isLocal = false, - isGlobbable = false - ) + val reader = ModuleReaderSpec("bird", true, false, false) val evaluatorId = client.sendCreateEvaluatorRequest(moduleReaders = listOf(reader)) client.send( EvaluateRequest( - requestId = 1, - evaluatorId = evaluatorId, - moduleUri = URI("bird:/pigeon.pkl"), - moduleText = null, - expr = "output.text", + 1, + evaluatorId, + URI("bird:/pigeon.pkl"), + null, + "output.text", ) ) @@ -836,22 +729,21 @@ abstract class AbstractServerTest { client.send( ReadModuleResponse( - requestId = readModuleRequest.requestId, - evaluatorId = evaluatorId, - contents = - """ + readModuleRequest.requestId, + evaluatorId, + """ firstName = "Pigeon" lastName = "Bird" fullName = firstName + " " + lastName """ - .trimIndent(), - error = null + .trimIndent(), + null ) ) val evaluateResponse = client.receive() assertThat(evaluateResponse.result).isNotNull - assertThat(evaluateResponse.result!!.debugYaml) + assertThat(evaluateResponse.result?.debugYaml) .isEqualTo( """ | @@ -865,33 +757,19 @@ abstract class AbstractServerTest { @Test fun `concurrent evaluations`() { - val reader = - ModuleReaderSpec( - scheme = "bird", - hasHierarchicalUris = true, - isLocal = false, - isGlobbable = false - ) + val reader = ModuleReaderSpec("bird", true, false, false) val evaluatorId = client.sendCreateEvaluatorRequest(moduleReaders = listOf(reader)) client.send( EvaluateRequest( - requestId = 1, - evaluatorId = evaluatorId, - moduleUri = URI("bird:/pigeon.pkl"), - moduleText = null, - expr = "output.text", + 1, + evaluatorId, + URI("bird:/pigeon.pkl"), + null, + "output.text", ) ) - client.send( - EvaluateRequest( - requestId = 2, - evaluatorId = evaluatorId, - moduleUri = URI("bird:/parrot.pkl"), - moduleText = null, - expr = "output.text" - ) - ) + client.send(EvaluateRequest(2, evaluatorId, URI("bird:/parrot.pkl"), null, "output.text")) // evaluation is single-threaded; `parrot.pkl` gets evaluated after `pigeon.pkl` completes. val response11 = client.receive() @@ -901,20 +779,19 @@ abstract class AbstractServerTest { ReadModuleResponse( response11.requestId, evaluatorId, - contents = - """ + """ firstName = "Pigeon" lastName = "Bird" fullName = firstName + " " + lastName """ - .trimIndent(), - error = null + .trimIndent(), + null ) ) val response12 = client.receive() assertThat(response12.result).isNotNull - assertThat(response12.result!!.debugYaml) + assertThat(response12.result?.debugYaml) .isEqualTo( """ | @@ -932,20 +809,19 @@ abstract class AbstractServerTest { ReadModuleResponse( response21.requestId, evaluatorId, - contents = - """ + """ firstName = "Parrot" lastName = "Bird" fullName = firstName + " " + lastName """ - .trimIndent(), - error = null + .trimIndent(), + null ) ) val response22 = client.receive() assertThat(response22.result).isNotNull - assertThat(response22.result!!.debugYaml) + assertThat(response22.result?.debugYaml) .isEqualTo( """ | @@ -1039,34 +915,32 @@ abstract class AbstractServerTest { cacheDir = cacheDir, project = Project( - projectFileUri = projectDir.resolve("PklProject").toUri(), - packageUri = null, - dependencies = - mapOf( - "birds" to - RemoteDependency(packageUri = URI("package://localhost:0/birds@0.5.0"), null), - "lib" to - Project( - projectFileUri = libDir.toUri().resolve("PklProject"), - packageUri = URI("package://localhost:0/lib@5.0.0"), - dependencies = emptyMap() - ) - ) + projectDir.resolve("PklProject").toUri(), + null, + mapOf( + "birds" to RemoteDependency(URI("package://localhost:0/birds@0.5.0"), null), + "lib" to + Project( + libDir.toUri().resolve("PklProject"), + URI("package://localhost:0/lib@5.0.0"), + emptyMap() + ) + ) ) ) client.send( EvaluateRequest( - requestId = 1, - evaluatorId = evaluatorId, - moduleUri = module.toUri(), - moduleText = null, - expr = "output.text", + 1, + evaluatorId, + module.toUri(), + null, + "output.text", ) ) val resp2 = client.receive() assertThat(resp2.error).isNull() assertThat(resp2.result).isNotNull() - assertThat(resp2.result!!.debugRendering.trim()) + assertThat(resp2.result?.debugRendering?.trim()) .isEqualTo( """ | @@ -1098,26 +972,28 @@ abstract class AbstractServerTest { ): Long { val message = CreateEvaluatorRequest( - requestId = 123, - allowedResources = listOf(Pattern.compile(".*")), - allowedModules = listOf(Pattern.compile(".*")), - clientResourceReaders = resourceReaders, - clientModuleReaders = moduleReaders, - modulePaths = modulePaths, - env = mapOf(), - properties = mapOf(), - timeout = null, - rootDir = null, - cacheDir = cacheDir, - outputFormat = null, - project = project, - http = http + 123, + listOf(Pattern.compile(".*")), + listOf(Pattern.compile(".*")), + moduleReaders, + resourceReaders, + modulePaths, + mapOf(), + mapOf(), + null, + null, + cacheDir, + null, + project, + http, + null, + null ) send(message) val response = receive() - assertThat(response.requestId).isEqualTo(requestId) + assertThat(response.requestId()).isEqualTo(requestId) assertThat(response.evaluatorId).isNotNull assertThat(response.error).isNull() diff --git a/pkl-server/src/test/kotlin/org/pkl/server/BinaryEvaluatorSnippetTests.kt b/pkl-server/src/test/kotlin/org/pkl/server/BinaryEvaluatorSnippetTests.kt index 6c9772ac4..a14ff93c0 100644 --- a/pkl-server/src/test/kotlin/org/pkl/server/BinaryEvaluatorSnippetTests.kt +++ b/pkl-server/src/test/kotlin/org/pkl/server/BinaryEvaluatorSnippetTests.kt @@ -66,3 +66,6 @@ class BinaryEvaluatorSnippetTestEngine : InputOutputTestEngine() { return true to bytes.debugRendering.stripFilePaths() } } + +val ByteArray.debugRendering + get() = MessagePackDebugRenderer(this).output diff --git a/pkl-server/src/test/kotlin/org/pkl/server/JvmServerTest.kt b/pkl-server/src/test/kotlin/org/pkl/server/JvmServerTest.kt index 189a9e87e..abe30444f 100644 --- a/pkl-server/src/test/kotlin/org/pkl/server/JvmServerTest.kt +++ b/pkl-server/src/test/kotlin/org/pkl/server/JvmServerTest.kt @@ -19,17 +19,32 @@ import java.io.PipedInputStream import java.io.PipedOutputStream import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.BeforeEach +import org.pkl.core.messaging.MessageTransport +import org.pkl.core.messaging.MessageTransports +import org.pkl.core.util.Pair class JvmServerTest : AbstractServerTest() { private val transports: Pair = run { if (USE_DIRECT_TRANSPORT) { - MessageTransports.direct() + MessageTransports.direct(::log) } else { - val in1 = PipedInputStream() + val in1 = + PipedInputStream(10240) // use larger pipe size since large messages can be >1024 bytes val out1 = PipedOutputStream(in1) val in2 = PipedInputStream() val out2 = PipedOutputStream(in2) - MessageTransports.stream(in1, out2) to MessageTransports.stream(in2, out1) + Pair.of( + MessageTransports.stream( + ServerMessagePackDecoder(in1), + ServerMessagePackEncoder(out2), + ::log + ), + MessageTransports.stream( + ServerMessagePackDecoder(in2), + ServerMessagePackEncoder(out1), + ::log + ) + ) } } diff --git a/pkl-server/src/test/kotlin/org/pkl/server/MessagePackDebugRenderer.kt b/pkl-server/src/test/kotlin/org/pkl/server/MessagePackDebugRenderer.kt index fe489bdbc..bf6d34c0e 100644 --- a/pkl-server/src/test/kotlin/org/pkl/server/MessagePackDebugRenderer.kt +++ b/pkl-server/src/test/kotlin/org/pkl/server/MessagePackDebugRenderer.kt @@ -105,6 +105,3 @@ class MessagePackDebugRenderer(bytes: ByteArray) { sb.toString().removePrefix("\n") } } - -val ByteArray.debugRendering - get() = MessagePackDebugRenderer(this).output diff --git a/pkl-server/src/test/kotlin/org/pkl/server/NativeServerTest.kt b/pkl-server/src/test/kotlin/org/pkl/server/NativeServerTest.kt index 303857c0c..40b7c69c8 100644 --- a/pkl-server/src/test/kotlin/org/pkl/server/NativeServerTest.kt +++ b/pkl-server/src/test/kotlin/org/pkl/server/NativeServerTest.kt @@ -18,6 +18,7 @@ package org.pkl.server import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.BeforeEach import org.pkl.commons.test.PklExecutablePaths +import org.pkl.core.messaging.MessageTransports class NativeServerTest : AbstractServerTest() { private lateinit var server: Process @@ -27,7 +28,14 @@ class NativeServerTest : AbstractServerTest() { fun beforeEach() { val executable = PklExecutablePaths.firstExisting.toString() server = ProcessBuilder(executable, "server").start() - client = TestTransport(MessageTransports.stream(server.inputStream, server.outputStream)) + client = + TestTransport( + MessageTransports.stream( + ServerMessagePackDecoder(server.inputStream), + ServerMessagePackEncoder(server.outputStream) + ) { _ -> + } + ) executor.execute { client.start() } } diff --git a/pkl-server/src/test/kotlin/org/pkl/server/MessagePackCodecTest.kt b/pkl-server/src/test/kotlin/org/pkl/server/ServerMessagePackCodecTest.kt similarity index 53% rename from pkl-server/src/test/kotlin/org/pkl/server/MessagePackCodecTest.kt rename to pkl-server/src/test/kotlin/org/pkl/server/ServerMessagePackCodecTest.kt index 77f9a98b0..26dd08cfc 100644 --- a/pkl-server/src/test/kotlin/org/pkl/server/MessagePackCodecTest.kt +++ b/pkl-server/src/test/kotlin/org/pkl/server/ServerMessagePackCodecTest.kt @@ -23,21 +23,25 @@ import java.time.Duration import java.util.regex.Pattern import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Test -import org.junit.jupiter.api.assertThrows import org.msgpack.core.MessagePack import org.pkl.core.evaluatorSettings.PklEvaluatorSettings -import org.pkl.core.module.PathElement +import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader +import org.pkl.core.messaging.Message +import org.pkl.core.messaging.MessageDecoder +import org.pkl.core.messaging.MessageEncoder +import org.pkl.core.messaging.Messages.* import org.pkl.core.packages.Checksums -class MessagePackCodecTest { +class ServerMessagePackCodecTest { private val encoder: MessageEncoder private val decoder: MessageDecoder init { - val inputStream = PipedInputStream() + val inputStream = + PipedInputStream(10240) // use larger pipe size since large messages can be >1024 bytes val outputStream = PipedOutputStream(inputStream) - encoder = MessagePackEncoder(MessagePack.newDefaultPacker(outputStream)) - decoder = MessagePackDecoder(MessagePack.newDefaultUnpacker(inputStream)) + encoder = ServerMessagePackEncoder(MessagePack.newDefaultPacker(outputStream)) + decoder = ServerMessagePackDecoder(MessagePack.newDefaultUnpacker(inputStream)) } private fun roundtrip(message: Message) { @@ -50,31 +54,19 @@ class MessagePackCodecTest { fun `round-trip CreateEvaluatorRequest`() { val resourceReader1 = ResourceReaderSpec( - scheme = "resourceReader1", - hasHierarchicalUris = true, - isGlobbable = true, + "resourceReader1", + true, + true, ) val resourceReader2 = ResourceReaderSpec( - scheme = "resourceReader2", - hasHierarchicalUris = true, - isGlobbable = false, + "resourceReader2", + true, + false, ) - val moduleReader1 = - ModuleReaderSpec( - scheme = "moduleReader1", - hasHierarchicalUris = true, - isGlobbable = true, - isLocal = true - ) - val moduleReader2 = - ModuleReaderSpec( - scheme = "moduleReader2", - hasHierarchicalUris = true, - isGlobbable = false, - isLocal = false - ) - @Suppress("HttpUrlsUsage") + val moduleReader1 = ModuleReaderSpec("moduleReader1", true, true, true) + val moduleReader2 = ModuleReaderSpec("moduleReader2", true, false, false) + val externalReader = ExternalReader("external-cmd", listOf("arg1", "arg2")) roundtrip( CreateEvaluatorRequest( requestId = 123, @@ -119,7 +111,9 @@ class MessagePackCodecTest { Http( proxy = PklEvaluatorSettings.Proxy(URI("http://foo.com:1234"), listOf("bar", "baz")), caCertificates = byteArrayOf(1, 2, 3, 4) - ) + ), + externalModuleReaders = mapOf("external" to externalReader, "external2" to externalReader), + externalResourceReaders = mapOf("external" to externalReader), ) ) } @@ -170,113 +164,4 @@ class MessagePackCodecTest { ) ) } - - @Test - fun `round-trip ReadResourceRequest`() { - roundtrip( - ReadResourceRequest(requestId = 123, evaluatorId = 456, uri = URI("some/resource.json")) - ) - } - - @Test - fun `round-trip ReadResourceResponse`() { - roundtrip( - ReadResourceResponse( - requestId = 123, - evaluatorId = 456, - contents = byteArrayOf(1, 2, 3, 4, 5), - error = null - ) - ) - } - - @Test - fun `round-trip ReadModuleRequest`() { - roundtrip(ReadModuleRequest(requestId = 123, evaluatorId = 456, uri = URI("some/module.pkl"))) - } - - @Test - fun `round-trip ReadModuleResponse`() { - roundtrip( - ReadModuleResponse(requestId = 123, evaluatorId = 456, contents = "x = 42", error = null) - ) - } - - @Test - fun `round-trip ListModulesRequest`() { - roundtrip(ListModulesRequest(requestId = 135, evaluatorId = 246, uri = URI("foo:/bar/baz/biz"))) - } - - @Test - fun `round-trip ListModulesResponse`() { - roundtrip( - ListModulesResponse( - requestId = 123, - evaluatorId = 234, - pathElements = listOf(PathElement("foo", true), PathElement("bar", false)), - error = null - ) - ) - roundtrip( - ListModulesResponse( - requestId = 123, - evaluatorId = 234, - pathElements = null, - error = "Something dun went wrong" - ) - ) - } - - @Test - fun `round-trip ListResourcesRequest`() { - roundtrip(ListResourcesRequest(requestId = 987, evaluatorId = 1359, uri = URI("bar:/bazzy"))) - } - - @Test - fun `round-trip ListResourcesResponse`() { - roundtrip( - ListResourcesResponse( - requestId = 3851, - evaluatorId = 3019, - pathElements = listOf(PathElement("foo", true), PathElement("bar", false)), - error = null - ) - ) - roundtrip( - ListResourcesResponse( - requestId = 3851, - evaluatorId = 3019, - pathElements = null, - error = "something went wrong" - ) - ) - } - - @Test - fun `decode request with missing request ID`() { - val bytes = - MessagePack.newDefaultBufferPacker() - .apply { - packArrayHeader(2) - packInt(MessageType.CREATE_EVALUATOR_REQUEST.code) - packMapHeader(1) - packString("clientResourceSchemes") - packArrayHeader(0) - } - .toByteArray() - - val decoder = MessagePackDecoder(MessagePack.newDefaultUnpacker(bytes)) - val exception = assertThrows { decoder.decode() } - assertThat(exception.message).contains("requestId") - } - - @Test - fun `decode invalid message header`() { - val bytes = MessagePack.newDefaultBufferPacker().apply { packInt(2) }.toByteArray() - - val decoder = MessagePackDecoder(MessagePack.newDefaultUnpacker(bytes)) - val exception = assertThrows { decoder.decode() } - assertThat(exception).hasMessage("Malformed message header.") - assertThat(exception).hasRootCauseMessage("Expected Array, but got Integer (02)") - } } diff --git a/pkl-server/src/test/kotlin/org/pkl/server/TestTransport.kt b/pkl-server/src/test/kotlin/org/pkl/server/TestTransport.kt index deed22747..5782c335e 100644 --- a/pkl-server/src/test/kotlin/org/pkl/server/TestTransport.kt +++ b/pkl-server/src/test/kotlin/org/pkl/server/TestTransport.kt @@ -17,7 +17,9 @@ package org.pkl.server import java.util.concurrent.ArrayBlockingQueue import java.util.concurrent.BlockingQueue -import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.Assertions.* +import org.pkl.core.messaging.Message +import org.pkl.core.messaging.MessageTransport class TestTransport(private val delegate: MessageTransport) : AutoCloseable { val incomingMessages: BlockingQueue = ArrayBlockingQueue(10) @@ -30,15 +32,15 @@ class TestTransport(private val delegate: MessageTransport) : AutoCloseable { delegate.close() } - fun send(message: ClientOneWayMessage) { + fun send(message: Message.Client.OneWay) { delegate.send(message) } - fun send(message: ClientRequestMessage) { + fun send(message: Message.Client.Request) { delegate.send(message) { incomingMessages.put(it) } } - fun send(message: ClientResponseMessage) { + fun send(message: Message.Client.Response) { delegate.send(message) } diff --git a/stdlib/EvaluatorSettings.pkl b/stdlib/EvaluatorSettings.pkl index 043886f14..7b50c562e 100644 --- a/stdlib/EvaluatorSettings.pkl +++ b/stdlib/EvaluatorSettings.pkl @@ -91,6 +91,14 @@ rootDir: String? /// Configuration of outgoing HTTP requests. http: Http? +/// Configuration for external module reader processes. +@Since { version = "0.27.0" } +externalModuleReaders: Mapping? + +/// Configuration for external resource reader processes. +@Since { version = "0.27.0" } +externalResourceReaders: Mapping? + /// Settings that control how Pkl talks to HTTP(S) servers. class Http { /// Configuration of the HTTP proxy to use. @@ -144,3 +152,20 @@ class Proxy { /// ``` noProxy: Listing(isDistinct) } + +@Since { version = "0.27.0" } +class ExternalReader { + /// The external reader executable. + /// + /// Will be spawned with the same environment variables and working directory as the Pkl process. + /// Executable is resolved according to the operating system's process spawning rules. + /// On macOS, Linux, and Windows platforms, this may be: + /// + /// * An absolute path + /// * A relative path (to the currrent working directory) + /// * The name of the executable, to be resolved against the `PATH` environment variable + executable: String + + /// Additional command line arguments passed to the external reader process. + arguments: Listing? +}