diff --git a/pom.xml b/pom.xml index 3542f94..e53e767 100644 --- a/pom.xml +++ b/pom.xml @@ -18,23 +18,25 @@ 11 11 11 + + 999-SNAPSHOT com.dylibso.chicory runtime - 0.0.12 + ${chicory.version} com.dylibso.chicory wasi - 0.0.12 + ${chicory.version} com.dylibso.chicory aot - 0.0.12 + ${chicory.version} junit diff --git a/src/main/java/org/extism/chicory/sdk/ChicoryModule.java b/src/main/java/org/extism/chicory/sdk/ChicoryModule.java new file mode 100644 index 0000000..4562b79 --- /dev/null +++ b/src/main/java/org/extism/chicory/sdk/ChicoryModule.java @@ -0,0 +1,46 @@ +package org.extism.chicory.sdk; + +import com.dylibso.chicory.aot.AotMachine; +import com.dylibso.chicory.runtime.Instance; +import com.dylibso.chicory.wasm.Module; +import com.dylibso.chicory.wasm.Parser; + +import java.nio.file.Path; + +class ChicoryModule { + + static final boolean IS_NATIVE_IMAGE_AOT = Boolean.getBoolean("com.oracle.graalvm.isaot"); + + static Module fromWasm(ManifestWasm m) { + if (m instanceof ManifestWasmBytes) { + ManifestWasmBytes mwb = (ManifestWasmBytes) m; + return Parser.parse(mwb.bytes); + } else if (m instanceof ManifestWasmPath) { + ManifestWasmPath mwp = (ManifestWasmPath) m; + return Parser.parse(Path.of(mwp.path)); + } else if (m instanceof ManifestWasmFile) { + ManifestWasmFile mwf = (ManifestWasmFile) m; + return Parser.parse(mwf.filePath); + } else if (m instanceof ManifestWasmUrl) { + ManifestWasmUrl mwu = (ManifestWasmUrl) m; + return Parser.parse(mwu.getUrlAsStream()); + } else { + throw new IllegalArgumentException("Unknown ManifestWasm type " + m.getClass()); + } + } + + static Instance.Builder instanceWithOptions(Module m, Manifest.Options opts) { + Instance.Builder builder = Instance.builder(m); + if (opts == null) { + return builder; + } + // This feature is not compatibly with the native-image builder. + if (opts.aot && !IS_NATIVE_IMAGE_AOT) { + builder.withMachineFactory(AotMachine::new); + } + if (!opts.validationFlags.isEmpty()) { + throw new UnsupportedOperationException("Validation flags are not supported yet"); + } + return builder; + } +} diff --git a/src/main/java/org/extism/chicory/sdk/DependencyGraph.java b/src/main/java/org/extism/chicory/sdk/DependencyGraph.java new file mode 100644 index 0000000..e8ae4a1 --- /dev/null +++ b/src/main/java/org/extism/chicory/sdk/DependencyGraph.java @@ -0,0 +1,351 @@ +package org.extism.chicory.sdk; + +import com.dylibso.chicory.log.Logger; +import com.dylibso.chicory.runtime.ExportFunction; +import com.dylibso.chicory.runtime.HostFunction; +import com.dylibso.chicory.runtime.HostImports; +import com.dylibso.chicory.runtime.Instance; +import com.dylibso.chicory.runtime.Store; +import com.dylibso.chicory.runtime.WasmFunctionHandle; +import com.dylibso.chicory.wasm.Module; +import com.dylibso.chicory.wasm.types.Export; +import com.dylibso.chicory.wasm.types.ExportSection; +import com.dylibso.chicory.wasm.types.ExternalType; +import com.dylibso.chicory.wasm.types.FunctionImport; +import com.dylibso.chicory.wasm.types.FunctionType; +import com.dylibso.chicory.wasm.types.Import; +import com.dylibso.chicory.wasm.types.ImportSection; +import com.dylibso.chicory.wasm.types.Value; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.Stack; + +import static java.util.stream.Collectors.groupingBy; + +class DependencyGraph { + public static final String MAIN_MODULE_NAME = "main"; + + private final Logger logger; + + private final Map> registeredSymbols = new HashMap<>(); + private final Map modules = new HashMap<>(); + private final Set hostModules = new HashSet<>(); + private final Map instances = new HashMap<>(); + private final Map trampolines = new HashMap<>(); + + private final Store store = new Store(); + private Manifest.Options options; + + public DependencyGraph(Logger logger) { + this.logger = logger; + } + + /** + * Set the instantiation options. + */ + public void setOptions(Manifest.Options options) { + this.options = options; + } + + /** + * Registers all the given named modules, and tries to look for a `main`. + *

+ * Try to find the main module: + * - There is always one main module + * - If a Wasm value has the Name field set to "main" then use that module + * - If there is only one module in the manifest then that is the main module by default + * - Otherwise the last module listed is the main module + */ + public void registerModules(ManifestWasm... wasms) { + for (int i = 0; i < wasms.length; i++) { + ManifestWasm wasm = wasms[i]; + boolean isLast = i == wasms.length - 1; + String moduleName = wasm.name; + Module m = ChicoryModule.fromWasm(wasm); + + if ((moduleName == null || moduleName.isEmpty() || isLast) + && !this.modules.containsKey(MAIN_MODULE_NAME)) { + moduleName = MAIN_MODULE_NAME; + } + + // TODO: checkHash(moduleName, wasm); + registerModule(moduleName, m); + + } + } + + private void checkCollision(String moduleName, String symbol) { + if (symbol == null && this.registeredSymbols.containsKey(moduleName)) { + throw new ExtismException("Collision detected: a module with the given name already exists: " + moduleName); + } else if (this.registeredSymbols.containsKey(moduleName) && this.registeredSymbols.get(moduleName).contains(symbol)) { + throw new ExtismException("Collision detected: a symbol with the given name already exists: " + moduleName + "." + symbol); + } + } + + /** + * Register a Module with the given name. + */ + public void registerModule(String name, Module m) { + checkCollision(name, null); + + ExportSection exportSection = m.exportSection(); + for (int i = 0; i < exportSection.exportCount(); i++) { + Export export = exportSection.getExport(i); + String exportName = export.name(); + this.registerSymbol(name, exportName); + } + modules.put(name, m); + } + + public void registerSymbol(String name, String symbol) { + checkCollision(name, symbol); + registeredSymbols.computeIfAbsent(name, k -> new HashSet<>()).add(symbol); + } + + public boolean validate() { + boolean valid = true; + for (var kv : modules.entrySet()) { + Module m = kv.getValue(); + + ImportSection imports = m.importSection(); + for (int i = 0; i < imports.importCount(); i++) { + Import imp = imports.getImport(i); + String moduleName = imp.moduleName(); + String symbolName = imp.name(); + if (!registeredSymbols.containsKey(moduleName) || !registeredSymbols.get(moduleName).contains(symbolName)) { + logger.warnf("Cannot find symbol: %s.%s\n", moduleName, symbolName); + valid = false; + } + if (!modules.containsKey(moduleName) && !hostModules.contains(moduleName)) { + logger.warnf("Cannot find definition for the given symbol: %s.%s\n", moduleName, symbolName); + valid = false; + } + } + } + return valid; + } + + /** + * Instantiate is a breadth-first visit of the dependency graph, starting + * from the `main` module, and recursively instantiating the required dependencies. + *

+ * The method is idempotent, invoking it twice causes it to return the same instance. + * + * @return an instance of the main module. + */ + public Instance instantiate() { + Instance mainInstance = this.getMainInstance(); + if (mainInstance != null) { + return mainInstance; + } + + if (!validate()) { + throw new ExtismException("Unresolved symbols"); + } + + Stack unresolved = new Stack<>(); + unresolved.push(MAIN_MODULE_NAME); + + while (!unresolved.isEmpty()) { + String moduleId = unresolved.peek(); + Module m = this.modules.get(moduleId); + boolean satisfied = true; + List trampolines = new ArrayList<>(); + ImportSection imports = m.importSection(); + // We assume that each unique `name` in an import of the form `name.symbol` + // is registered as a module with that name + // + // FIXME: this is actually a strong assumption, because we could + // define "overrides" by overwriting individual `name.symbol` in our table. + var requiredModules = imports.stream().collect(groupingBy(Import::moduleName)); + + if (!requiredModules.isEmpty()) { + // We need to check whether the given import is available + for (String requiredModule : requiredModules.keySet()) { + if (unresolved.contains(requiredModule)) { + // This is a cycle! + var moduleImports = requiredModules.get(requiredModule); + for (Import mi : moduleImports) { + if (mi.importType() == ExternalType.FUNCTION) { + // It's ok, we just add one little indirection. + // This will be resolved at the end, when everything is settled. + trampolines.add(registerTrampoline((FunctionImport) mi, m)); + } else { + throw new ExtismException("cycle detected on a non-function"); + } + } + } else if (!this.instances.containsKey(requiredModule) && !this.hostModules.contains(requiredModule)) { + // No such instance nor registered host function; we schedule this module for visiting. + satisfied = false; + unresolved.push(requiredModule); + } + } + } + + // The store already contains everything we need, + // we can proceed with pop the name from the stack + // and instantiate. + if (satisfied) { + unresolved.pop(); + instantiate(moduleId, trampolines); + } + } + + // We are now ready to resolve all the trampolines. + for (var t : trampolines.entrySet()) { + QualifiedName name = t.getKey(); + Trampoline trampoline = t.getValue(); + + ExportFunction ef = instances.get(name.moduleName).export(name.fieldName); + trampoline.resolveFunction(ef); + } + + // We can now initialize all modules. + for (var inst : this.instances.values()) { + inst.initialize(true); + } + + return this.getMainInstance(); + } + + private Instance instantiate(String moduleId, List moreHostFunctions) { + Module m = this.modules.get(moduleId); + Objects.requireNonNull(m); + + HostImports extendedHostImports = + mergeHostImports(store.toHostImports(), moreHostFunctions); + + Instance instance = + ChicoryModule.instanceWithOptions(m, this.options) + .withHostImports(extendedHostImports) + .withStart(false) + .build(); + this.store.register(moduleId, instance); + this.instances.put(moduleId, instance); + return instance; + } + + private HostImports mergeHostImports(HostImports hostImports, List trampolines) { + HostFunction[] hostFunctions = hostImports.functions(); + List mergedList = new ArrayList<>(trampolines); + for (HostFunction fn : hostFunctions) { + for (HostFunction t : trampolines) { + if (t.moduleName().equals(fn.fieldName()) && t.fieldName().equals(fn.fieldName())) { + // If one such case exists, the "proper" function takes precedence over the trampoline. + mergedList.remove(t); + } + } + mergedList.add(fn); + } + return new HostImports( + mergedList.toArray(new HostFunction[mergedList.size()]), + hostImports.globals(), + hostImports.memories(), + hostImports.tables()); + } + + private HostFunction registerTrampoline(FunctionImport f, Module m) { + // Trampolines are singletons for each pair. + // Trampolines are not registered into the store, as they are not "real" functions. + // They are instead kept separately and passed explicitly to the instance. + Trampoline trampoline = this.trampolines.computeIfAbsent( + new QualifiedName(f.moduleName(), f.name()), k -> new Trampoline()); + var functionType = m.typeSection().getType(f.typeIndex()); + return trampoline.asHostFunction(f.moduleName(), f.name(), functionType); + } + + /** + * Register the given host functions in the store. Each host function + * has a "module name" and a symbol name, thus we register each module name + * in the "hostModules" set. + */ + public void registerFunctions(HostFunction... functions) { + store.addFunction(functions); + for (HostFunction f : functions) { + this.hostModules.add(f.moduleName()); + registerSymbol(f.moduleName(), f.fieldName()); + } + } + + /** + * @return a named instance with the given name. The method is idempotent, + * invoking it twice causes it to return the same instance. + */ + public Instance getInstance(String moduleName) { + if (instances.containsKey(moduleName)) { + return instances.get(moduleName); + } else { + return instantiate(moduleName, List.of()); + } + } + + /** + * @return the main instance. + */ + private Instance getMainInstance() { + return this.instances.get(MAIN_MODULE_NAME); + } + + static final class Trampoline implements WasmFunctionHandle { + WasmFunctionHandle f = + (Instance instance, Value... args) -> { + throw new ExtismException("Unresolved trampoline"); + }; + + public void resolveFunction(HostFunction hf) { + this.f = hf.handle(); + } + + public void resolveFunction(ExportFunction ef) { + this.f = (Instance instance, Value... args) -> ef.apply(args); + } + + @Override + public Value[] apply(Instance instance, Value... args) { + return f.apply(instance, args); + } + + public HostFunction asHostFunction(String moduleName, String name, FunctionType functionType) { + return new HostFunction(this, moduleName, name, + functionType.params(), functionType.returns()); + } + } + + /** + * A pair moduleName, symbol name. + */ + static final class QualifiedName { + final String moduleName; + final String fieldName; + + public QualifiedName(String moduleName, String fieldName) { + this.moduleName = moduleName; + this.fieldName = fieldName; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof QualifiedName)) { + return false; + } + QualifiedName qualifiedName = (QualifiedName) o; + return Objects.equals(moduleName, qualifiedName.moduleName) + && Objects.equals(fieldName, qualifiedName.fieldName); + } + + @Override + public int hashCode() { + return Objects.hash(moduleName, fieldName); + } + } + +} \ No newline at end of file diff --git a/src/main/java/org/extism/chicory/sdk/Kernel.java b/src/main/java/org/extism/chicory/sdk/Kernel.java index 7bbd2b3..7379e50 100644 --- a/src/main/java/org/extism/chicory/sdk/Kernel.java +++ b/src/main/java/org/extism/chicory/sdk/Kernel.java @@ -1,22 +1,15 @@ package org.extism.chicory.sdk; -import static com.dylibso.chicory.wasm.types.Value.*; - -import com.dylibso.chicory.aot.AotMachine; -import com.dylibso.chicory.log.Logger; -import com.dylibso.chicory.log.SystemLogger; import com.dylibso.chicory.runtime.ExportFunction; -import com.dylibso.chicory.runtime.HostFunction; import com.dylibso.chicory.runtime.Instance; -import com.dylibso.chicory.runtime.Module; -import com.dylibso.chicory.wasm.types.Value; -import com.dylibso.chicory.wasm.types.ValueType; import com.dylibso.chicory.runtime.Memory; -import java.util.HashMap; -import java.util.List; +import com.dylibso.chicory.wasm.Module; +import com.dylibso.chicory.wasm.Parser; + +import static com.dylibso.chicory.wasm.types.Value.i64; public class Kernel { - private static final String IMPORT_MODULE_NAME = "extism:host/env"; + public static final String IMPORT_MODULE_NAME = "extism:host/env"; private final Memory memory; private final ExportFunction alloc; private final ExportFunction free; @@ -39,19 +32,7 @@ public class Kernel { private final ExportFunction errorGet; private final ExportFunction memoryBytes; - public Kernel() { - this(new SystemLogger()); - } - - public Kernel(Logger logger) { - var kernelStream = getClass().getClassLoader().getResourceAsStream("extism-runtime.wasm"); - - var moduleBuilder = Module.builder(kernelStream).withLogger(logger); - - // uncomment for AOT mode - //moduleBuilder = moduleBuilder.withMachineFactory(AotMachine::new); - - Instance kernel = moduleBuilder.build().instantiate(); + public Kernel(Instance kernel) { memory = kernel.memory(); alloc = kernel.export("alloc"); free = kernel.export("free"); @@ -75,6 +56,11 @@ public Kernel(Logger logger) { memoryBytes = kernel.export("memory_bytes"); } + public static Module module() { + var kernelStream = Kernel.class.getClassLoader().getResourceAsStream("extism-runtime.wasm"); + return Parser.parse(kernelStream); + } + public void setInput(byte[] input) { var ptr = alloc.apply(i64(input.length))[0]; memory.write(ptr.asInt(), input); @@ -86,218 +72,4 @@ public byte[] getOutput() { var len = outputLen.apply()[0]; return memory.readBytes(ptr.asInt(), len.asInt()); } - - public HostFunction[] toHostFunctions() { - var hostFunctions = new HostFunction[23]; - int count = 0; - - hostFunctions[count++] = - new HostFunction( - (Instance instance, Value... args) -> alloc.apply(args), - IMPORT_MODULE_NAME, - "alloc", - List.of(ValueType.I64), - List.of(ValueType.I64)); - - hostFunctions[count++] = - new HostFunction( - (Instance instance, Value... args) -> free.apply(args), - IMPORT_MODULE_NAME, - "free", - List.of(ValueType.I64), - List.of()); - - hostFunctions[count++] = - new HostFunction( - (Instance instance, Value... args) -> length.apply(args), - IMPORT_MODULE_NAME, - "length", - List.of(ValueType.I64), - List.of(ValueType.I64)); - - hostFunctions[count++] = - new HostFunction( - (Instance instance, Value... args) -> lengthUnsafe.apply(args), - IMPORT_MODULE_NAME, - "length_unsafe", - List.of(ValueType.I64), - List.of(ValueType.I64)); - - hostFunctions[count++] = - new HostFunction( - (Instance instance, Value... args) -> loadU8.apply(args), - IMPORT_MODULE_NAME, - "load_u8", - List.of(ValueType.I64), - List.of(ValueType.I32)); - - hostFunctions[count++] = - new HostFunction( - (Instance instance, Value... args) -> loadU64.apply(args), - IMPORT_MODULE_NAME, - "load_u64", - List.of(ValueType.I64), - List.of(ValueType.I64)); - - hostFunctions[count++] = - new HostFunction( - (Instance instance, Value... args) -> inputLoadU8.apply(args), - IMPORT_MODULE_NAME, - "input_load_u8", - List.of(ValueType.I64), - List.of(ValueType.I32)); - - hostFunctions[count++] = - new HostFunction( - (Instance instance, Value... args) -> inputLoadU64.apply(args), - IMPORT_MODULE_NAME, - "input_load_u64", - List.of(ValueType.I64), - List.of(ValueType.I64)); - - hostFunctions[count++] = - new HostFunction( - (Instance instance, Value... args) -> storeU8.apply(args), - IMPORT_MODULE_NAME, - "store_u8", - List.of(ValueType.I64, ValueType.I32), - List.of()); - - hostFunctions[count++] = - new HostFunction( - (Instance instance, Value... args) -> storeU64.apply(args), - IMPORT_MODULE_NAME, - "store_u64", - List.of(ValueType.I64, ValueType.I64), - List.of()); - - hostFunctions[count++] = - new HostFunction( - (Instance instance, Value... args) -> inputSet.apply(args), - IMPORT_MODULE_NAME, - "input_set", - List.of(ValueType.I64, ValueType.I64), - List.of()); - - hostFunctions[count++] = - new HostFunction( - (Instance instance, Value... args) -> inputLen.apply(args), - IMPORT_MODULE_NAME, - "input_length", - List.of(), - List.of(ValueType.I64)); - - hostFunctions[count++] = - new HostFunction( - (Instance instance, Value... args) -> inputOffset.apply(args), - IMPORT_MODULE_NAME, - "input_offset", - List.of(), - List.of(ValueType.I64)); - - hostFunctions[count++] = - new HostFunction( - (Instance instance, Value... args) -> outputSet.apply(args), - IMPORT_MODULE_NAME, - "output_set", - List.of(ValueType.I64, ValueType.I64), - List.of()); - - hostFunctions[count++] = - new HostFunction( - (Instance instance, Value... args) -> outputLen.apply(args), - IMPORT_MODULE_NAME, - "output_length", - List.of(), - List.of(ValueType.I64)); - - hostFunctions[count++] = - new HostFunction( - (Instance instance, Value... args) -> outputOffset.apply(args), - IMPORT_MODULE_NAME, - "output_offset", - List.of(), - List.of(ValueType.I64)); - - hostFunctions[count++] = - new HostFunction( - (Instance instance, Value... args) -> reset.apply(args), - IMPORT_MODULE_NAME, - "reset", - List.of(), - List.of()); - - hostFunctions[count++] = - new HostFunction( - (Instance instance, Value... args) -> errorSet.apply(args), - IMPORT_MODULE_NAME, - "error_set", - List.of(ValueType.I64), - List.of()); - - hostFunctions[count++] = - new HostFunction( - (Instance instance, Value... args) -> errorGet.apply(args), - IMPORT_MODULE_NAME, - "error_get", - List.of(), - List.of(ValueType.I64)); - - hostFunctions[count++] = - new HostFunction( - (Instance instance, Value... args) -> memoryBytes.apply(args), - IMPORT_MODULE_NAME, - "memory_bytes", - List.of(), - List.of(ValueType.I64)); - - var vars = new HashMap(); - - hostFunctions[count++] = - new HostFunction( - (Instance instance, Value... args) -> { - // System.out.println("_var_get " + args); - // var keyLen = Length.apply(args[0])[0]; - // var key = memory.getString(args[0].asInt(), - // keyLen.asInt()); - // var value = vars.get(key); - return new Value[] {i64(0)}; - }, - IMPORT_MODULE_NAME, - "var_get", - List.of(ValueType.I64, ValueType.I64), - List.of(ValueType.I64)); - - hostFunctions[count++] = - new HostFunction( - (Instance instance, Value... args) -> { - // System.out.println("_var_set" + args); - // var keyLen = Length.apply(args[0])[0]; - // var key = memory.getString(args[0].asInt(), - // keyLen.asInt()); - // var value = vars.get(key); - return null; - }, - IMPORT_MODULE_NAME, - "var_set", - List.of(ValueType.I64, ValueType.I64), - List.of()); - - hostFunctions[count++] = - new HostFunction( - (Instance instance, Value... args) -> { - // System.out.println("_config_get" + args); - // var keyLen = Length.apply(args[0])[0]; - // var key = memory.getString(args[0].asInt(), - // keyLen.asInt()); - // var value = vars.get(key); - return new Value[] {i64(0)}; - }, - IMPORT_MODULE_NAME, - "config_get", - List.of(ValueType.I64), - List.of(ValueType.I64)); - - return hostFunctions; - } } diff --git a/src/main/java/org/extism/chicory/sdk/Linker.java b/src/main/java/org/extism/chicory/sdk/Linker.java new file mode 100644 index 0000000..0cf1b65 --- /dev/null +++ b/src/main/java/org/extism/chicory/sdk/Linker.java @@ -0,0 +1,53 @@ +package org.extism.chicory.sdk; + +import com.dylibso.chicory.log.Logger; +import com.dylibso.chicory.runtime.HostFunction; +import com.dylibso.chicory.runtime.Instance; +import com.dylibso.chicory.wasi.WasiPreview1; + + +/** + * Links together the modules in the given manifest with the given host functions + * and predefined support modules (e.g. the {@link Kernel}. + *

+ * Returns a {@link Plugin}. + */ +class Linker { + public static final String EXTISM_NS = "extism:host/env"; + private final Manifest manifest; + private final HostFunction[] hostFunctions; + private final Logger logger; + + Linker(Manifest manifest, HostFunction[] hostFunctions, Logger logger) { + this.manifest = manifest; + this.hostFunctions = hostFunctions; + this.logger = logger; + } + + public Plugin link() { + + var dg = new DependencyGraph(logger); + dg.setOptions(manifest.options); + + // Register the Kernel module, usually not present in the manifest. + dg.registerModule(Kernel.IMPORT_MODULE_NAME, Kernel.module()); + + // Register the WASI host functions. + dg.registerFunctions(new WasiPreview1(logger).toHostFunctions()); + + // Register the user-provided host functions. + dg.registerFunctions(this.hostFunctions); + + // Register all the modules declared in the manifest. + dg.registerModules(manifest.wasms); + + // Instantiate the main module, and, recursively, all of its dependencies. + Instance main = dg.instantiate(); + // The kernel has been now instantiated, get a handle for it. + Instance kernelInstance = dg.getInstance(Kernel.IMPORT_MODULE_NAME); + + return new Plugin(main, new Kernel(kernelInstance)); + } + +} + diff --git a/src/main/java/org/extism/chicory/sdk/ManifestModuleMapper.java b/src/main/java/org/extism/chicory/sdk/ManifestModuleMapper.java deleted file mode 100644 index c729ea1..0000000 --- a/src/main/java/org/extism/chicory/sdk/ManifestModuleMapper.java +++ /dev/null @@ -1,53 +0,0 @@ -package org.extism.chicory.sdk; - -import com.dylibso.chicory.aot.AotMachine; -import com.dylibso.chicory.runtime.Module; - -class ManifestModuleMapper { - private final Manifest manifest; - - ManifestModuleMapper(Manifest manifest) { - this.manifest = manifest; - } - - Module.Builder toModuleBuilder() { - if (manifest.wasms.length > 1) { - throw new UnsupportedOperationException( - "Manifests of multiple wasm files are not supported yet!"); - } - Module.Builder mb = wasmToModuleBuilder(manifest.wasms[0]); - return withOptions(mb, manifest.options); - } - - private Module.Builder wasmToModuleBuilder(ManifestWasm m) { - if (m instanceof ManifestWasmBytes) { - ManifestWasmBytes mwb = (ManifestWasmBytes) m; - return Module.builder(mwb.bytes); - } else if (m instanceof ManifestWasmPath) { - ManifestWasmPath mwp = (ManifestWasmPath) m; - return Module.builder(mwp.path); - } else if (m instanceof ManifestWasmFile) { - ManifestWasmFile mwf = (ManifestWasmFile) m; - return Module.builder(mwf.filePath); - } else if (m instanceof ManifestWasmUrl) { - ManifestWasmUrl mwu = (ManifestWasmUrl) m; - return Module.builder(mwu.getUrlAsStream()); - } else { - throw new IllegalArgumentException("Unknown ManifestWasm type " + m.getClass()); - } - } - - private Module.Builder withOptions(Module.Builder mb, Manifest.Options opts) { - if (opts == null) { - return mb; - } - if (opts.aot) { - mb.withMachineFactory(AotMachine::new); - } - if (!opts.validationFlags.isEmpty()) { - throw new UnsupportedOperationException("Validation flags are not supported yet"); - } - return mb; - } - -} diff --git a/src/main/java/org/extism/chicory/sdk/Plugin.java b/src/main/java/org/extism/chicory/sdk/Plugin.java index 99b753c..ea5d374 100644 --- a/src/main/java/org/extism/chicory/sdk/Plugin.java +++ b/src/main/java/org/extism/chicory/sdk/Plugin.java @@ -3,12 +3,18 @@ import com.dylibso.chicory.log.Logger; import com.dylibso.chicory.log.SystemLogger; import com.dylibso.chicory.runtime.HostFunction; -import com.dylibso.chicory.runtime.HostImports; import com.dylibso.chicory.runtime.Instance; -import com.dylibso.chicory.wasi.WasiOptions; -import com.dylibso.chicory.wasi.WasiPreview1; +/** + * A Plugin instance. + * + * Plugins can be instantiated using a {@link Plugin.Builder}, returned + * by {@link Plugin#ofManifest(Manifest)}. The Builder allows to set options + * on the Plugin, such as {@link HostFunction}s and the {@link Logger}. + * + */ public class Plugin { + public static Builder ofManifest(Manifest manifest) { return new Builder(manifest); } @@ -34,55 +40,24 @@ public Builder withLogger(Logger logger) { } public Plugin build() { - return new Plugin(manifest, hostFunctions, logger); + var logger = this.logger == null ? new SystemLogger() : this.logger; + Linker linker = new Linker(this.manifest, this.hostFunctions, logger); + return linker.link(); } } - private final Manifest manifest; - private final Instance instance; - private final HostImports imports; private final Kernel kernel; - private Plugin(Manifest manifest) { - this(manifest, new HostFunction[]{}, null); - } - - private Plugin(Manifest manifest, HostFunction[] hostFunctions, Logger logger) { - if (logger == null) { - logger = new SystemLogger(); - } - - this.kernel = new Kernel(logger); - this.manifest = manifest; - - // TODO: Expand WASI Support here - var options = WasiOptions.builder().build(); - var wasi = new WasiPreview1(logger, options); - var wasiHostFunctions = wasi.toHostFunctions(); + private final Instance mainInstance; - var hostFuncList = getHostFunctions(kernel.toHostFunctions(), hostFunctions, wasiHostFunctions); - this.imports = new HostImports(hostFuncList); - - var moduleBuilder = new ManifestModuleMapper(manifest) - .toModuleBuilder() - .withLogger(logger) - .withHostImports(imports); - - this.instance = moduleBuilder.build().instantiate(); - } - - private static HostFunction[] getHostFunctions( - HostFunction[] kernelFuncs, HostFunction[] hostFunctions, HostFunction[] wasiHostFunctions) { - // concat list of host functions - var hostFuncList = new HostFunction[hostFunctions.length + kernelFuncs.length + wasiHostFunctions.length]; - System.arraycopy(kernelFuncs, 0, hostFuncList, 0, kernelFuncs.length); - System.arraycopy(hostFunctions, 0, hostFuncList, kernelFuncs.length, hostFunctions.length); - System.arraycopy(wasiHostFunctions, 0, hostFuncList, kernelFuncs.length + hostFunctions.length, wasiHostFunctions.length); - return hostFuncList; + Plugin(Instance main, Kernel kernel) { + this.kernel = kernel; + this.mainInstance = main; + mainInstance.initialize(true); } public byte[] call(String funcName, byte[] input) { - var func = instance.export(funcName); + var func = mainInstance.export(funcName); kernel.setInput(input); var result = func.apply()[0].asInt(); if (result == 0) { @@ -91,5 +66,4 @@ public byte[] call(String funcName, byte[] input) { throw new ExtismException("Failed"); } } - } diff --git a/src/test/java/org/extism/chicory/sdk/DependencyGraphTest.java b/src/test/java/org/extism/chicory/sdk/DependencyGraphTest.java new file mode 100644 index 0000000..ab9d305 --- /dev/null +++ b/src/test/java/org/extism/chicory/sdk/DependencyGraphTest.java @@ -0,0 +1,102 @@ +package org.extism.chicory.sdk; + +import com.dylibso.chicory.log.SystemLogger; +import com.dylibso.chicory.runtime.Instance; +import com.dylibso.chicory.wasi.WasiPreview1; +import com.dylibso.chicory.wasm.Module; +import com.dylibso.chicory.wasm.Parser; +import com.dylibso.chicory.wasm.types.Value; +import junit.framework.TestCase; + +import java.io.IOException; +import java.io.InputStream; + +public class DependencyGraphTest extends TestCase { + + public void testCircularDeps() throws IOException { + InputStream is1 = this.getClass().getResourceAsStream("/circular-import/circular-import-1.wasm"); + InputStream is2 = this.getClass().getResourceAsStream("/circular-import/circular-import-2.wasm"); + InputStream is3 = this.getClass().getResourceAsStream("/circular-import/circular-import-main.wasm"); + + DependencyGraph dg = new DependencyGraph(new SystemLogger()); + + dg.registerModule("env-1", Parser.parse(is1.readAllBytes())); + dg.registerModule("env-2", Parser.parse(is2.readAllBytes())); + dg.registerModule("main", Parser.parse(is3.readAllBytes())); + + Instance main = dg.instantiate(); + + Value[] result = main.export("real_do_expr").apply(); + assertEquals(60, result[0].asInt()); + } + + public void testCircularDepsMore() throws IOException { + InputStream addBytes = this.getClass().getResourceAsStream("/circular-import-more/circular-import-add.wasm"); + InputStream subBytes = this.getClass().getResourceAsStream("/circular-import-more/circular-import-sub.wasm"); + InputStream exprBytes = this.getClass().getResourceAsStream("/circular-import-more/circular-import-expr.wasm"); + InputStream mainBytes = this.getClass().getResourceAsStream("/circular-import-more/circular-import-main.wasm"); + + + Module add = Parser.parse(addBytes.readAllBytes()); + Module sub = Parser.parse(subBytes.readAllBytes()); + Module expr = Parser.parse(exprBytes.readAllBytes()); + Module main = Parser.parse(mainBytes.readAllBytes()); + + { + DependencyGraph dg = new DependencyGraph(new SystemLogger()); + dg.registerModule("add", add); + dg.registerModule("sub", sub); + dg.registerModule("expr", expr); + dg.registerModule("main", main); + + Instance mainInst = dg.instantiate(); + + Value[] result = mainInst.export("real_do_expr").apply(); + assertEquals(60, result[0].asInt()); + } + + // Let's try to register them in a different order: + // it should never matter. + { + DependencyGraph dg = new DependencyGraph(new SystemLogger()); + dg.registerModule("expr", expr); + dg.registerModule("main", main); + dg.registerModule("sub", sub); + dg.registerModule("add", add); + + Instance mainInst = dg.instantiate(); + + Value[] result = mainInst.export("real_do_expr").apply(); + assertEquals(60, result[0].asInt()); + } + } + + public void testHostFunctionDeps() throws IOException { + InputStream requireWasi = this.getClass().getResourceAsStream("/host-functions/import-wasi.wasm"); + Module requireWasiM = Parser.parse(requireWasi.readAllBytes()); + + DependencyGraph dg = new DependencyGraph(new SystemLogger()); + dg.registerFunctions(WasiPreview1.builder().build().toHostFunctions()); + dg.registerModule("main", requireWasiM); + + // The host functions should be found, thus the module should not be further searched in the DependencyGraph. + // If the search did not stop, it would cause an error, because there is no actual module to instantiate. + Instance mainInst = dg.instantiate(); + assertNotNull(mainInst); + } + + public void testInstantiate() throws IOException { + InputStream requireWasi = this.getClass().getResourceAsStream("/host-functions/import-wasi.wasm"); + Module requireWasiM = Parser.parse(requireWasi.readAllBytes()); + + DependencyGraph dg = new DependencyGraph(new SystemLogger()); + dg.registerFunctions(WasiPreview1.builder().build().toHostFunctions()); + dg.registerModule("main", requireWasiM); + + Instance mainInst = dg.instantiate(); + Instance mainInst2 = dg.instantiate(); + assertSame("when invoked twice, instantiate() returns the same instance", mainInst, mainInst2); + } + + +} diff --git a/src/test/java/org/extism/chicory/sdk/PluginTest.java b/src/test/java/org/extism/chicory/sdk/PluginTest.java index a1c7133..94dbb04 100644 --- a/src/test/java/org/extism/chicory/sdk/PluginTest.java +++ b/src/test/java/org/extism/chicory/sdk/PluginTest.java @@ -33,4 +33,5 @@ public void testGreetAoT() { assertEquals("Hello, Benjamin!", result); } + } diff --git a/src/test/resources/circular-import-more/circular-import-add.wasm b/src/test/resources/circular-import-more/circular-import-add.wasm new file mode 100644 index 0000000..10c439e Binary files /dev/null and b/src/test/resources/circular-import-more/circular-import-add.wasm differ diff --git a/src/test/resources/circular-import-more/circular-import-add.wat b/src/test/resources/circular-import-more/circular-import-add.wat new file mode 100644 index 0000000..ff31e5f --- /dev/null +++ b/src/test/resources/circular-import-more/circular-import-add.wat @@ -0,0 +1,15 @@ +(module + + (import "expr" "expr" (func $expr (result i32))) + + (func $add (export "add") (param i32 i32) (result i32) + (i32.add + (local.get 0) + (local.get 1)) + ) + + (func $do_expr (export "do_expr") (result i32) + call $expr + ) + +) diff --git a/src/test/resources/circular-import-more/circular-import-expr.wasm b/src/test/resources/circular-import-more/circular-import-expr.wasm new file mode 100644 index 0000000..0f36df6 Binary files /dev/null and b/src/test/resources/circular-import-more/circular-import-expr.wasm differ diff --git a/src/test/resources/circular-import-more/circular-import-expr.wat b/src/test/resources/circular-import-more/circular-import-expr.wat new file mode 100644 index 0000000..4a9d44f --- /dev/null +++ b/src/test/resources/circular-import-more/circular-import-expr.wat @@ -0,0 +1,14 @@ +(module + + (import "add" "add" (func $add (param i32 i32) (result i32))) + (import "sub" "sub" (func $sub (param i32 i32) (result i32))) + + (func $expr (export "expr") (result i32) + (i32.const 20) + (i32.const 50) + (call $add) + (i32.const 10) + (call $sub) + ) + +) diff --git a/src/test/resources/circular-import-more/circular-import-main.wasm b/src/test/resources/circular-import-more/circular-import-main.wasm new file mode 100644 index 0000000..df6f46a Binary files /dev/null and b/src/test/resources/circular-import-more/circular-import-main.wasm differ diff --git a/src/test/resources/circular-import-more/circular-import-main.wat b/src/test/resources/circular-import-more/circular-import-main.wat new file mode 100644 index 0000000..294c258 --- /dev/null +++ b/src/test/resources/circular-import-more/circular-import-main.wat @@ -0,0 +1,8 @@ +(module + + (import "add" "do_expr" (func $do_expr (result i32))) + + (func $real_do_expr (export "real_do_expr") (result i32) + (call $do_expr) + ) +) \ No newline at end of file diff --git a/src/test/resources/circular-import-more/circular-import-sub.wasm b/src/test/resources/circular-import-more/circular-import-sub.wasm new file mode 100644 index 0000000..6b242e2 Binary files /dev/null and b/src/test/resources/circular-import-more/circular-import-sub.wasm differ diff --git a/src/test/resources/circular-import-more/circular-import-sub.wat b/src/test/resources/circular-import-more/circular-import-sub.wat new file mode 100644 index 0000000..35d9a58 --- /dev/null +++ b/src/test/resources/circular-import-more/circular-import-sub.wat @@ -0,0 +1,9 @@ +(module + + (func $sub (export "sub") (param i32 i32) (result i32) + (i32.sub + (local.get 0) + (local.get 1)) + ) + +) diff --git a/src/test/resources/circular-import/circular-import-1.wasm b/src/test/resources/circular-import/circular-import-1.wasm new file mode 100644 index 0000000..8266694 Binary files /dev/null and b/src/test/resources/circular-import/circular-import-1.wasm differ diff --git a/src/test/resources/circular-import/circular-import-1.wat b/src/test/resources/circular-import/circular-import-1.wat new file mode 100644 index 0000000..0aee507 --- /dev/null +++ b/src/test/resources/circular-import/circular-import-1.wat @@ -0,0 +1,14 @@ +(module + + (import "env-2" "add" (func $add (param i32 i32) (result i32))) + (import "env-2" "sub" (func $sub (param i32 i32) (result i32))) + + (func $expr (export "expr") (result i32) + (i32.const 20) + (i32.const 50) + (call $add) + (i32.const 10) + (call $sub) + ) + +) diff --git a/src/test/resources/circular-import/circular-import-2.wasm b/src/test/resources/circular-import/circular-import-2.wasm new file mode 100644 index 0000000..75b1eca Binary files /dev/null and b/src/test/resources/circular-import/circular-import-2.wasm differ diff --git a/src/test/resources/circular-import/circular-import-2.wat b/src/test/resources/circular-import/circular-import-2.wat new file mode 100644 index 0000000..d83361f --- /dev/null +++ b/src/test/resources/circular-import/circular-import-2.wat @@ -0,0 +1,22 @@ +(module + + (import "env-1" "expr" (func $expr (result i32))) + + (func $add (export "add") (param i32 i32) (result i32) + (i32.add + (local.get 0) + (local.get 1)) + ) + + (func $sub (export "sub") (param i32 i32) (result i32) + (i32.sub + (local.get 0) + (local.get 1)) + ) + + (func $do_expr (export "do_expr") (result i32) + call $expr + ) + + +) diff --git a/src/test/resources/circular-import/circular-import-main.wasm b/src/test/resources/circular-import/circular-import-main.wasm new file mode 100644 index 0000000..3fb28c6 Binary files /dev/null and b/src/test/resources/circular-import/circular-import-main.wasm differ diff --git a/src/test/resources/circular-import/circular-import-main.wat b/src/test/resources/circular-import/circular-import-main.wat new file mode 100644 index 0000000..c4fd33b --- /dev/null +++ b/src/test/resources/circular-import/circular-import-main.wat @@ -0,0 +1,8 @@ +(module + + (import "env-2" "do_expr" (func $do_expr (result i32))) + + (func $real_do_expr (export "real_do_expr") (result i32) + (call $do_expr) + ) +) \ No newline at end of file diff --git a/src/test/resources/host-functions/import-wasi.wasm b/src/test/resources/host-functions/import-wasi.wasm new file mode 100644 index 0000000..8db9bec Binary files /dev/null and b/src/test/resources/host-functions/import-wasi.wasm differ diff --git a/src/test/resources/host-functions/import-wasi.wat b/src/test/resources/host-functions/import-wasi.wat new file mode 100644 index 0000000..98c5271 --- /dev/null +++ b/src/test/resources/host-functions/import-wasi.wat @@ -0,0 +1,7 @@ +(module + (import "wasi_snapshot_preview1" "fd_write" (func $fd_write (param i32 i32 i32 i32) (result i32))) + + (memory 1) + (export "memory" (memory 0)) + + (func $main (export "_start") nop)) \ No newline at end of file