Skip to content

Commit

Permalink
Add ExternalReaderRuntime, lots of test coverage
Browse files Browse the repository at this point in the history
  • Loading branch information
HT154 committed Oct 11, 2024
1 parent ca16d00 commit a935e9d
Show file tree
Hide file tree
Showing 22 changed files with 672 additions and 79 deletions.
4 changes: 2 additions & 2 deletions bench/src/jmh/java/org/pkl/core/ListSort.java
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
import org.pkl.core.repl.ReplRequest;
import org.pkl.core.repl.ReplResponse;
import org.pkl.core.repl.ReplServer;
import org.pkl.core.resource.ResourceReaderFactories;
import org.pkl.core.resource.ResourceReaders;
import org.pkl.core.util.IoUtils;

@Warmup(iterations = 5, time = 2)
Expand All @@ -43,7 +43,7 @@ public class ListSort {
HttpClient.dummyClient(),
Loggers.stdErr(),
List.of(ModuleKeyFactories.standardLibrary),
List.of(ResourceReaderFactories.file()),
List.of(ResourceReaders.file()),
Map.of(),
Map.of(),
null,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,7 @@ abstract class CliCommand(protected val cliOptions: CliBaseOptions) {
// the same spec
// this avoids spawning multiple subprocesses if the same reader implements both reader types
// and/or multiple schemes
(externalModuleReaders + externalResourceReaders).values.associateWith {
(externalModuleReaders + externalResourceReaders).values.toSet().associateWith {
ExternalProcessImpl(it)
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.assertThrows
import org.pkl.commons.cli.commands.BaseCommand
import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader

class BaseCommandTest {

Expand Down Expand Up @@ -72,4 +73,34 @@ class BaseCommandTest {

assertThat(cmd.baseOptions.allowedResources).isEmpty()
}

@Test
fun `--external-resource and --external-module are parsed correctly`() {
cmd.parse(
arrayOf(
"--external-module",
"scheme3=reader3",
"--external-module",
"scheme4=reader4 with args",
"--external-resource",
"scheme1=reader1",
"--external-resource",
"scheme2=reader2 with args"
)
)
assertThat(cmd.baseOptions.externalModuleReaders)
.isEqualTo(
mapOf(
"scheme3" to ExternalReader("reader3", emptyList()),
"scheme4" to ExternalReader("reader4", listOf("with", "args"))
)
)
assertThat(cmd.baseOptions.externalResourceReaders)
.isEqualTo(
mapOf(
"scheme1" to ExternalReader("reader1", emptyList()),
"scheme2" to ExternalReader("reader2", listOf("with", "args"))
)
)
}
}
15 changes: 11 additions & 4 deletions pkl-core/src/main/java/org/pkl/core/EvaluatorBuilder.java
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
import java.util.*;
import java.util.regex.Pattern;
import org.pkl.core.SecurityManagers.StandardBuilder;
import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader;
import org.pkl.core.externalProcess.ExternalProcess;
import org.pkl.core.externalProcess.ExternalProcessImpl;
import org.pkl.core.http.HttpClient;
import org.pkl.core.module.ModuleKeyFactories;
Expand Down Expand Up @@ -475,16 +477,21 @@ public EvaluatorBuilder applyFromProject(Project project) {
} else if (settings.moduleCacheDir() != null) {
setModuleCacheDir(settings.moduleCacheDir());
}

// this isn't ideal as project and non-project ExternalProcessImpl instances can be dupes
var procs = new HashMap<ExternalReader, ExternalProcess>();
if (settings.externalModuleReaders() != null) {
for (var entry : settings.externalModuleReaders().entrySet()) {
var process = new ExternalProcessImpl(entry.getValue());
addModuleKeyFactory(ModuleKeyFactories.external(entry.getKey(), process));
addModuleKeyFactory(
ModuleKeyFactories.external(
entry.getKey(), procs.computeIfAbsent(entry.getValue(), ExternalProcessImpl::new)));
}
}
if (settings.externalResourceReaders() != null) {
for (var entry : settings.externalResourceReaders().entrySet()) {
var process = new ExternalProcessImpl(entry.getValue());
addResourceReader(ResourceReaders.external(entry.getKey(), process));
addResourceReader(
ResourceReaders.external(
entry.getKey(), procs.computeIfAbsent(entry.getValue(), ExternalProcessImpl::new)));
}
}
return this;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
/**
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.core.externalProcess;

import java.net.URI;
import org.pkl.core.messaging.Messages.ModuleReaderSpec;

public interface ExternalModuleReader extends ExternalReaderBase {
boolean isLocal();

String read(URI uri) throws Exception;

default ModuleReaderSpec getSpec() {
return new ModuleReaderSpec(getScheme(), hasHierarchicalUris(), isLocal(), isGlobbable());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ public ExternalProcessMessagePackEncoder(OutputStream outputStream) {
}
case INITIALIZE_MODULE_READER_RESPONSE -> {
var m = (InitializeModuleReaderResponse) msg;
packer.packMapHeader(2);
packMapHeader(1, m.getSpec());
packKeyValue("requestId", m.getRequestId());
if (m.getSpec() != null) {
packer.packString("spec");
Expand All @@ -61,7 +61,7 @@ public ExternalProcessMessagePackEncoder(OutputStream outputStream) {
}
case INITIALIZE_RESOURCE_READER_RESPONSE -> {
var m = (InitializeResourceReaderResponse) msg;
packer.packMapHeader(2);
packMapHeader(1, m.getSpec());
packKeyValue("requestId", m.getRequestId());
if (m.getSpec() != null) {
packer.packString("spec");
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
/**
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.core.externalProcess;

import java.net.URI;
import java.util.List;
import org.pkl.core.module.PathElement;

public interface ExternalReaderBase {
String getScheme();

boolean hasHierarchicalUris();

boolean isGlobbable();

List<PathElement> listElements(URI uri) throws Exception;
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,191 @@
/**
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.core.externalProcess;

import java.io.IOException;
import java.util.List;
import org.pkl.core.externalProcess.ExternalProcessMessages.*;
import org.pkl.core.messaging.Message.Type;
import org.pkl.core.messaging.MessageTransport;
import org.pkl.core.messaging.Messages.*;
import org.pkl.core.messaging.ProtocolException;
import org.pkl.core.util.Nullable;

public class ExternalReaderRuntime {

private final List<ExternalModuleReader> moduleReaders;
private final List<ExternalResourceReader> resourceReaders;
private final MessageTransport transport;

public ExternalReaderRuntime(
List<ExternalModuleReader> moduleReaders,
List<ExternalResourceReader> resourceReaders,
MessageTransport transport) {
this.moduleReaders = moduleReaders;
this.resourceReaders = resourceReaders;
this.transport = transport;
}

public void close() {
transport.close();
}

private @Nullable ExternalModuleReader findModuleReader(String scheme) {
for (var moduleReader : moduleReaders) {
if (moduleReader.getScheme().equalsIgnoreCase(scheme)) {
return moduleReader;
}
}
return null;
}

private @Nullable ExternalResourceReader findResourceReader(String scheme) {
for (var resourceReader : resourceReaders) {
if (resourceReader.getScheme().equalsIgnoreCase(scheme)) {
return resourceReader;
}
}
return null;
}

public void run() throws ProtocolException, IOException {
transport.start(
(msg) -> {
if (msg.getType() == Type.CLOSE_EXTERNAL_PROCESS) {
close();
} else {
throw new ProtocolException("Unexpected incoming one-way message: " + msg);
}
},
(msg) -> {
switch (msg.getType()) {
case INITIALIZE_MODULE_READER_REQUEST -> {
var req = (InitializeModuleReaderRequest) msg;
var reader = findModuleReader(req.getScheme());
@Nullable ModuleReaderSpec spec = null;
if (reader != null) {
spec = reader.getSpec();
}
transport.send(new InitializeModuleReaderResponse(req.getRequestId(), spec));
}
case INITIALIZE_RESOURCE_READER_REQUEST -> {
var req = (InitializeResourceReaderRequest) msg;
var reader = findResourceReader(req.getScheme());
@Nullable ResourceReaderSpec spec = null;
if (reader != null) {
spec = reader.getSpec();
}
transport.send(new InitializeResourceReaderResponse(req.getRequestId(), spec));
}
case LIST_MODULES_REQUEST -> {
var req = (ListModulesRequest) msg;
var reader = findModuleReader(req.getUri().getScheme());
if (reader == null) {
transport.send(
new ListModulesResponse(
req.getRequestId(),
req.getEvaluatorId(),
null,
"No module reader found for scheme " + req.getUri().getScheme()));
return;
}
try {
transport.send(
new ListModulesResponse(
req.getRequestId(),
req.getEvaluatorId(),
reader.listElements(req.getUri()),
null));
} catch (Exception e) {
transport.send(
new ListModulesResponse(
req.getRequestId(), req.getEvaluatorId(), null, e.toString()));
}
}
case LIST_RESOURCES_REQUEST -> {
var req = (ListResourcesRequest) msg;
var reader = findModuleReader(req.getUri().getScheme());
if (reader == null) {
transport.send(
new ListResourcesResponse(
req.getRequestId(),
req.getEvaluatorId(),
null,
"No resource reader found for scheme " + req.getUri().getScheme()));
return;
}
try {
transport.send(
new ListResourcesResponse(
req.getRequestId(),
req.getEvaluatorId(),
reader.listElements(req.getUri()),
null));
} catch (Exception e) {
transport.send(
new ListResourcesResponse(
req.getRequestId(), req.getEvaluatorId(), null, e.toString()));
}
}
case READ_MODULE_REQUEST -> {
var req = (ReadModuleRequest) msg;
var reader = findModuleReader(req.getUri().getScheme());
if (reader == null) {
transport.send(
new ReadModuleResponse(
req.getRequestId(),
req.getEvaluatorId(),
null,
"No module reader found for scheme " + req.getUri().getScheme()));
return;
}
try {
transport.send(
new ReadModuleResponse(
req.getRequestId(), req.getEvaluatorId(), reader.read(req.getUri()), null));
} catch (Exception e) {
transport.send(
new ReadModuleResponse(
req.getRequestId(), req.getEvaluatorId(), null, e.toString()));
}
}
case READ_RESOURCE_REQUEST -> {
var req = (ReadResourceRequest) msg;
var reader = findResourceReader(req.getUri().getScheme());
if (reader == null) {
transport.send(
new ReadResourceResponse(
req.getRequestId(),
req.getEvaluatorId(),
null,
"No resource reader found for scheme " + req.getUri().getScheme()));
return;
}
try {
transport.send(
new ReadResourceResponse(
req.getRequestId(), req.getEvaluatorId(), reader.read(req.getUri()), null));
} catch (Exception e) {
transport.send(
new ReadResourceResponse(
req.getRequestId(), req.getEvaluatorId(), new byte[0], e.toString()));
}
}
default -> throw new ProtocolException("Unexpected incoming request message: " + msg);
}
});
}
}
Loading

0 comments on commit a935e9d

Please sign in to comment.