Implement SPICE-0009 External Readers (#660)

This adds a new feature, which allows Pkl to read resources and modules from external processes.

Follows the design laid out in SPICE-0009.

Also, this moves most of the messaging API into pkl-core

bench/gradle.lockfile

@@ -32,6 +32,7 @@ org.junit.jupiter:junit-jupiter-params:5.11.2=testCompileClasspath,testImplement
 org.junit.platform:junit-platform-commons:1.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.junit.platform:junit-platform-engine:1.11.2=testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.junit:junit-bom:5.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
+org.msgpack:msgpack-core:0.9.8=jmh,jmhRuntimeClasspath
 org.openjdk.jmh:jmh-core:1.37=jmh,jmhCompileClasspath,jmhImplementationDependenciesMetadata,jmhRuntimeClasspath
 org.openjdk.jmh:jmh-generator-asm:1.37=jmh,jmhCompileClasspath,jmhImplementationDependenciesMetadata,jmhRuntimeClasspath
 org.openjdk.jmh:jmh-generator-bytecode:1.37=jmh,jmhCompileClasspath,jmhImplementationDependenciesMetadata,jmhRuntimeClasspath

docs/gradle.lockfile

@@ -30,6 +30,7 @@ org.junit.jupiter:junit-jupiter-params:5.11.2=testCompileClasspath,testImplement
 org.junit.platform:junit-platform-commons:1.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.junit.platform:junit-platform-engine:1.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.junit:junit-bom:5.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
+org.msgpack:msgpack-core:0.9.8=testRuntimeClasspath
 org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.organicdesign:Paguro:3.10.3=testRuntimeClasspath
 org.snakeyaml:snakeyaml-engine:2.5=testRuntimeClasspath

docs/modules/bindings-specification/pages/message-passing-api.adoc

@@ -9,8 +9,8 @@ The first element of the array is a code that designates the message's type, enc
 The second element of the array is the message body, encoded as a map.
 
 Messages are passed between the _client_ and the _server_.
-The _client_ is the host language (for example, the Swift application when using pkl-swift).
-The _server_ is the entity that provides controls for interacting with Pkl.
+When hosting Pkl (for example, the Swift application when using pkl-swift), the _client_ is the host program and the _server_ is the entity that provides controls for interacting with Pkl.
+When implementing an xref:language-reference:index.adoc#external-readers[external reader], the _client_ is the external reader process and the _server_ is the Pkl evaluator.
 
 For example, in JSON representation:
 
@@ -597,3 +597,96 @@ class PathElement {
   isDirectory: Boolean
 }
 ----
+
+[[initialize-module-reader-request]]
+=== Initialize Module Reader Request
+
+Code: `0x2e` +
+Type: <<server-message,Server>> <<request-message,Request>>
+
+Initialize an xref:language-reference:index.adoc#external-readers[External Module Reader].
+This message is sent to external reader processes the first time a module scheme it is registered for is read.
+
+[source,pkl]
+----
+/// A number identifying this request.
+requestId: Int
+
+/// The module scheme to initialize.
+scheme: String
+----
+
+[[initialize-module-reader-response]]
+=== Initialize Module Reader Response
+
+Code: `0x2f` +
+Type: <<client-message,Client>> <<response-message,Response>>
+
+Return the requested external module reader specification.
+The `spec` field should be set to `null` when the external process does not implement the requested module scheme.
+
+[source,pkl]
+----
+/// A number identifying this request.
+requestId: Int
+
+/// Client-side module reader spec.
+///
+/// Null when the external process does not implement the requested module scheme.
+spec: ClientModuleReader?
+----
+
+`ClientModuleReader` is defined above by <<create-evaluator-request,Create Evaluator Request>>.
+
+[[initialize-resource-reader-request]]
+=== Initialize Resource Reader Request
+
+Code: `0x30` +
+Type: <<server-message,Server>> <<request-message,Request>>
+
+Initialize an xref:language-reference:index.adoc#external-readers[External Resource Reader].
+This message is sent to external reader processes the first time a resource scheme it is registered for is read.
+
+[source,pkl]
+----
+/// A number identifying this request.
+requestId: Int
+
+/// The resource scheme to initialize.
+scheme: String
+----
+
+[[initialize-resource-reader-response]]
+=== Initialize Resource Reader Response
+
+Code: `0x31` +
+Type: <<client-message,Client>> <<response-message,Response>>
+
+Return the requested external resource reader specification.
+The `spec` field should be set to `null` when the external process does not implement the requested resource scheme.
+
+[source,pkl]
+----
+/// A number identifying this request.
+requestId: Int
+
+/// Client-side resource reader spec.
+///
+/// Null when the external process does not implement the requested resource scheme.
+spec: ClientResourceReader?
+----
+
+`ClientResourceReader` is defined above by <<create-evaluator-request,Create Evaluator Request>>. 
+
+[[close-external-process]]
+=== Close External Process
+
+Code: `0x32` +
+Type: <<server-message,Server>> <<one-way-message,One Way>> 
+
+Initiate graceful shutdown of the external reader process.
+
+[source,pkl]
+----
+/// This message has no properties.
+----

docs/modules/language-reference/pages/index.adoc

@@ -3199,7 +3199,8 @@ This section discusses language features that are generally more relevant to tem
 <<grammar-definition,Grammar Definition>> +
 <<reserved-keywords,Reserved Keywords>> +
 <<blank-identifiers,Blank Identifiers>> +
-<<projects,Projects>>
+<<projects,Projects>> +
+<<external-readers,External Readers>>
 
 [[meaning-of-new]]
 === Meaning of `new`
@@ -5465,3 +5466,68 @@ It can be imported using dependency notation, i.e. `import "@fruit/Pear.pkl"`.
 At runtime, it will resolve to relative path `../fruit/Pear.pkl`.
 
 When packaging projects with local dependencies, both the project and its dependent project must be passed to the xref:pkl-cli:index.adoc#command-project-package[`pkl project package`] command.
+
+[[external-readers]]
+=== External Readers
+
+External readers are a mechanism to extend the <<modules,module>> and <<resources,resource>> URI schemes that Pkl supports.
+Readers are implemented as ordinary executables and use Pkl's xref:bindings-specification:message-passing-api.adoc[message passing API] to communicate with the hosting Pkl evaluator.
+The xref:swift:ROOT:index.adoc[Swift] and xref:go:ROOT:index.adoc[Go] language binding libraries provide an `ExternalReaderRuntime` type to facilitate implementing external readers.
+
+External readers are configured separately for modules and resources.
+They are registered by mapping their URI scheme to the executable to run and additonal arguments to pass.
+This is done on the command line by passing `--external-resource-reader` and `--external-module-reader` flags, which may both be passed multiple times.
+
+[source,text]
+----
+$ pkl eval <module> --external-resource-reader <scheme>=<executable> --external-module-reader <scheme>='<executable> <argument> <argument>'
+----
+
+External readers may also be configured in a <<projects, Project's>> `PklProject` file.
+[source,{pkl}]
+----
+evaluatorSettings {
+  externalResourceReaders {
+    ["<scheme>"] {
+      executable = "<executable>"
+    }
+  }
+  externalModuleReaders {
+    ["<scheme>"] {
+      executable = "<executable>"
+      arguments { "<arg>"; "<arg>" }
+    }
+  }
+}
+----
+
+Registering an external reader for a scheme automatically adds that scheme to the default allowed modules/resources.
+As with Pkl's built-in module and resource schemes, setting explicit allowed modules or resources overrides this behavior and appropriate patterns must be specified to allow use of external readers.
+
+==== Example
+
+Consider this module:
+
+[source,{pkl}]
+----
+username = "pigeon"
+
+email = read("ldap://ds.example.com:389/dc=example,dc=com?mail?sub?(uid=\(username))").text
+----
+
+Pkl doesn't implement the `ldap:` resource URI scheme natively, but an external reader can provide it.
+Assuming a hypothetical `pkl-ldap` executable implementing the external reader protocol and the `ldap:` scheme is in the `$PATH`, this module can be evaluated as:
+
+[source,text]
+----
+$ pkl eval <module> --external-resource-reader ldap=pkl-ldap
+username = "pigeon"
+email = "[email protected]"
+----
+
+In this example, the external reader may provide both `ldap:` and `ldaps:` schemes.
+To support both schemes during evaluation, both would need to be registered explicitly:
+[source,text]
+----
+$ pkl eval <module> --external-resource-reader ldap=pkl-ldap --external-resource-reader ldaps=pkl-ldap
+----

docs/src/test/kotlin/DocSnippetTests.kt

@@ -22,10 +22,10 @@ import org.pkl.core.parser.antlr.PklParser
 import org.pkl.core.repl.ReplRequest
 import org.pkl.core.repl.ReplResponse
 import org.pkl.core.repl.ReplServer
-import org.pkl.core.resource.ResourceReaders
 import org.pkl.core.util.IoUtils
 import org.antlr.v4.runtime.ParserRuleContext
 import org.pkl.core.http.HttpClient
+import org.pkl.core.resource.ResourceReaders
 import java.nio.file.Files
 import kotlin.io.path.isDirectory
 import kotlin.io.path.isRegularFile

pkl-cli/gradle.lockfile

@@ -99,4 +99,4 @@ org.xmlunit:xmlunit-core:2.10.0=testCompileClasspath,testImplementationDependenc
 org.xmlunit:xmlunit-legacy:2.10.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 org.xmlunit:xmlunit-placeholders:2.10.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 org.yaml:snakeyaml:2.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
-empty=annotationProcessor,intransitiveDependenciesMetadata,javaExecutable,kotlinCompilerPluginClasspath,kotlinNativeCompilerPluginClasspath,kotlinScriptDef,kotlinScriptDefExtensions,runtimeOnlyDependenciesMetadata,shadow,signatures,sourcesJar,stagedAlpineLinuxAmd64Executable,stagedLinuxAarch64Executable,stagedLinuxAmd64Executable,stagedMacAarch64Executable,stagedMacAmd64Executable,stagedWindowsAmd64Executable,testAnnotationProcessor,testApiDependenciesMetadata,testCompileOnlyDependenciesMetadata,testIntransitiveDependenciesMetadata,testKotlinScriptDef,testKotlinScriptDefExtensions
+empty=annotationProcessor,archives,compile,intransitiveDependenciesMetadata,javaExecutable,kotlinCompilerPluginClasspath,kotlinNativeCompilerPluginClasspath,kotlinScriptDef,kotlinScriptDefExtensions,runtime,runtimeOnlyDependenciesMetadata,shadow,signatures,sourcesJar,stagedAlpineLinuxAmd64Executable,stagedLinuxAarch64Executable,stagedLinuxAmd64Executable,stagedMacAarch64Executable,stagedMacAmd64Executable,stagedWindowsAmd64Executable,testAnnotationProcessor,testApiDependenciesMetadata,testCompile,testCompileOnly,testCompileOnlyDependenciesMetadata,testIntransitiveDependenciesMetadata,testKotlinScriptDef,testKotlinScriptDefExtensions,testRuntime

pkl-cli/src/main/kotlin/org/pkl/cli/CliEvaluator.kt

@@ -31,12 +31,12 @@ import org.pkl.commons.writeString
 import org.pkl.core.EvaluatorBuilder
 import org.pkl.core.ModuleSource
 import org.pkl.core.PklException
-import org.pkl.core.module.ModuleKeyFactories
 import org.pkl.core.module.ModulePathResolver
 import org.pkl.core.runtime.ModuleResolver
 import org.pkl.core.runtime.VmException
 import org.pkl.core.runtime.VmUtils
 import org.pkl.core.util.IoUtils
+import org.pkl.core.util.Readers
 
 private data class OutputFile(val pathSpec: String, val moduleUri: URI)
 
@@ -100,7 +100,8 @@ constructor(
         writeOutput(builder)
       }
     } finally {
-      ModuleKeyFactories.closeQuietly(builder.moduleKeyFactories)
+      Readers.closeQuietly(builder.moduleKeyFactories)
+      Readers.closeQuietly(builder.resourceReaders)
     }
   }
 

pkl-cli/src/main/kotlin/org/pkl/cli/CliImportAnalyzer.kt

@@ -20,7 +20,7 @@ import org.pkl.commons.cli.CliCommand
 import org.pkl.commons.createParentDirectories
 import org.pkl.commons.writeString
 import org.pkl.core.ModuleSource
-import org.pkl.core.module.ModuleKeyFactories
+import org.pkl.core.util.Readers
 
 class CliImportAnalyzer
 @JvmOverloads
@@ -73,7 +73,8 @@ constructor(
         .build()
         .use { it.evaluateOutputText(sourceModule) }
     } finally {
-      ModuleKeyFactories.closeQuietly(builder.moduleKeyFactories)
+      Readers.closeQuietly(builder.moduleKeyFactories)
+      Readers.closeQuietly(builder.resourceReaders)
     }
   }
 }

pkl-cli/src/main/kotlin/org/pkl/cli/CliServer.kt

@@ -18,14 +18,13 @@ package org.pkl.cli
 import org.pkl.commons.cli.CliBaseOptions
 import org.pkl.commons.cli.CliCommand
 import org.pkl.commons.cli.CliException
-import org.pkl.server.MessageTransports
-import org.pkl.server.ProtocolException
+import org.pkl.core.messaging.ProtocolException
 import org.pkl.server.Server
 
 class CliServer(options: CliBaseOptions) : CliCommand(options) {
   override fun doRun() =
     try {
-      val server = Server(MessageTransports.stream(System.`in`, System.out))
+      val server = Server.stream(System.`in`, System.out)
       server.use { it.start() }
     } catch (e: ProtocolException) {
       throw CliException(e.message!!)

pkl-cli/src/main/kotlin/org/pkl/cli/CliTestRunner.kt

@@ -19,10 +19,10 @@ import java.io.Writer
 import org.pkl.commons.cli.*
 import org.pkl.core.EvaluatorBuilder
 import org.pkl.core.ModuleSource.uri
-import org.pkl.core.module.ModuleKeyFactories
 import org.pkl.core.stdlib.test.report.JUnitReport
 import org.pkl.core.stdlib.test.report.SimpleReport
 import org.pkl.core.util.ErrorMessages
+import org.pkl.core.util.Readers
 
 class CliTestRunner
 @JvmOverloads
@@ -38,7 +38,8 @@ constructor(
     try {
       evalTest(builder)
     } finally {
-      ModuleKeyFactories.closeQuietly(builder.moduleKeyFactories)
+      Readers.closeQuietly(builder.moduleKeyFactories)
+      Readers.closeQuietly(builder.resourceReaders)
     }
   }
 

pkl-codegen-java/gradle.lockfile

@@ -33,6 +33,7 @@ org.junit.jupiter:junit-jupiter-params:5.11.2=testCompileClasspath,testImplement
 org.junit.platform:junit-platform-commons:1.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.junit.platform:junit-platform-engine:1.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.junit:junit-bom:5.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
+org.msgpack:msgpack-core:0.9.8=runtimeClasspath,testRuntimeClasspath
 org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.organicdesign:Paguro:3.10.3=runtimeClasspath,testRuntimeClasspath
 org.snakeyaml:snakeyaml-engine:2.5=runtimeClasspath,testRuntimeClasspath

pkl-codegen-java/src/main/kotlin/org/pkl/codegen/java/CliJavaCodeGenerator.kt

@@ -21,7 +21,7 @@ import org.pkl.commons.cli.CliException
 import org.pkl.commons.createParentDirectories
 import org.pkl.commons.writeString
 import org.pkl.core.ModuleSource
-import org.pkl.core.module.ModuleKeyFactories
+import org.pkl.core.util.Readers
 
 /** API for the Java code generator CLI. */
 class CliJavaCodeGenerator(private val options: CliJavaCodeGeneratorOptions) :
@@ -49,7 +49,8 @@ class CliJavaCodeGenerator(private val options: CliJavaCodeGeneratorOptions) :
         }
       }
     } finally {
-      ModuleKeyFactories.closeQuietly(builder.moduleKeyFactories)
+      Readers.closeQuietly(builder.moduleKeyFactories)
+      Readers.closeQuietly(builder.resourceReaders)
     }
   }
 }

pkl-codegen-kotlin/gradle.lockfile

@@ -36,6 +36,7 @@ org.junit.jupiter:junit-jupiter-params:5.11.2=testCompileClasspath,testImplement
 org.junit.platform:junit-platform-commons:1.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.junit.platform:junit-platform-engine:1.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.junit:junit-bom:5.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
+org.msgpack:msgpack-core:0.9.8=runtimeClasspath,testRuntimeClasspath
 org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.organicdesign:Paguro:3.10.3=runtimeClasspath,testRuntimeClasspath
 org.snakeyaml:snakeyaml-engine:2.5=runtimeClasspath,testRuntimeClasspath

pkl-codegen-kotlin/src/main/kotlin/org/pkl/codegen/kotlin/CliKotlinCodeGenerator.kt

@@ -21,7 +21,7 @@ import org.pkl.commons.cli.CliException
 import org.pkl.commons.createParentDirectories
 import org.pkl.commons.writeString
 import org.pkl.core.ModuleSource
-import org.pkl.core.module.ModuleKeyFactories
+import org.pkl.core.util.Readers
 
 /** API for the Kotlin code generator CLI. */
 class CliKotlinCodeGenerator(private val options: CliKotlinCodeGeneratorOptions) :
@@ -50,7 +50,8 @@ class CliKotlinCodeGenerator(private val options: CliKotlinCodeGeneratorOptions)
         }
       }
     } finally {
-      ModuleKeyFactories.closeQuietly(builder.moduleKeyFactories)
+      Readers.closeQuietly(builder.moduleKeyFactories)
+      Readers.closeQuietly(builder.resourceReaders)
     }
   }
 }

pkl-commons-cli/gradle.lockfile

@@ -31,6 +31,7 @@ org.junit.jupiter:junit-jupiter-params:5.11.2=testCompileClasspath,testImplement
 org.junit.platform:junit-platform-commons:1.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.junit.platform:junit-platform-engine:1.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.junit:junit-bom:5.11.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
+org.msgpack:msgpack-core:0.9.8=runtimeClasspath,testRuntimeClasspath
 org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath,testRuntimeOnlyDependenciesMetadata
 org.organicdesign:Paguro:3.10.3=runtimeClasspath,testRuntimeClasspath
 org.snakeyaml:snakeyaml-engine:2.5=runtimeClasspath,testRuntimeClasspath

pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliBaseOptions.kt

@@ -20,6 +20,7 @@ import java.nio.file.Files
 import java.nio.file.Path
 import java.time.Duration
 import java.util.regex.Pattern
+import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader
 import org.pkl.core.module.ProjectDependenciesManager
 import org.pkl.core.util.IoUtils
 
@@ -134,6 +135,12 @@ data class CliBaseOptions(
 
   /** Hostnames, IP addresses, or CIDR blocks to not proxy. */
   val httpNoProxy: List<String>? = null,
+
+  /** External module reader process specs */
+  val externalModuleReaders: Map<String, ExternalReader> = mapOf(),
+
+  /** External resource reader process specs */
+  val externalResourceReaders: Map<String, ExternalReader> = mapOf(),
 ) {
 
   companion object {