feat: passing wasm as via ptr and len in manifest, revamp extism::Wasm

src/extism.hpp

@@ -2,13 +2,15 @@
 
 #include <cstdint>
 #include <extism.h>
+#include <filesystem>
 #include <functional>
 #include <map>
 #include <memory>
 #include <optional>
 #include <stdexcept>
 #include <string>
 #include <string_view>
+#include <variant>
 #include <vector>
 
 namespace extism {
@@ -21,20 +23,73 @@ class Error : public std::runtime_error {
 
 typedef std::map<std::string, std::string> Config;
 
-enum WasmSource { WasmSourcePath, WasmSourceURL, WasmSourceBytes };
+class WasmBytes {
+  using DataSource =
+      std::variant<std::vector<uint8_t>, std::shared_ptr<const uint8_t[]>>;
+  DataSource src;
+  size_t srcSize;
 
-class Wasm {
-  WasmSource source;
-  std::string ref;
+public:
+  WasmBytes(std::shared_ptr<const uint8_t[]> src, const size_t srcSize)
+      : src(std::move(src)), srcSize(srcSize) {}
+  WasmBytes(std::vector<uint8_t> data)
+      : src(std::move(data)),
+        srcSize(std::get<std::vector<uint8_t>>(this->src).size()) {}
+  WasmBytes(const uint8_t *src, const size_t srcSize)
+      : WasmBytes(std::vector<uint8_t>(src, src + srcSize)) {}
+  // DANGEROUS, src must remain valid
+  static WasmBytes CreateWithoutOwnership(const uint8_t *src,
+                                          const size_t srcSize) {
+    return WasmBytes(std::shared_ptr<const uint8_t[]>(
+                         std::shared_ptr<const uint8_t[]>{}, src),
+                     srcSize);
+  }
+  const uint8_t *get() const {
+    if (std::holds_alternative<std::vector<uint8_t>>(src)) {
+      return std::get<std::vector<uint8_t>>(this->src).data();
+    }
+    return std::get<std::shared_ptr<const uint8_t[]>>(src).get();
+  }
+
+  size_t getSize() const { return srcSize; }
+};
+
+class WasmURL {
+public:
+  std::string url;
   std::string httpMethod;
   std::map<std::string, std::string> httpHeaders;
+  WasmURL(std::string url, std::string httpMethod = "GET",
+          std::map<std::string, std::string> httpHeaders = {})
+      : url(std::move(url)), httpMethod(std::move(httpMethod)),
+        httpHeaders(std::move(httpHeaders)) {}
+};
+
+enum WasmSource { WasmSourcePath, WasmSourceURL, WasmSourceBytes };
+
+class Wasm {
+  using WasmSourceType =
+      std::variant<std::filesystem::path, WasmURL, WasmBytes>;
 
-  // TODO: add base64 encoded raw data
+  WasmSourceType src;
   std::string _hash;
 
 public:
+  __attribute__((deprecated))
   Wasm(WasmSource source, std::string ref, std::string hash = std::string())
-      : source(source), ref(std::move(ref)), _hash(std::move(hash)) {}
+      : _hash(std::move(hash)) {
+    if (source == WasmSourceBytes) {
+      throw std::runtime_error(
+          "WasmSourceBytes not supported from this constructor");
+    } else if (source == WasmSourcePath) {
+      src = std::filesystem::path(ref);
+    } else if (source == WasmSourceURL) {
+      src = WasmURL(ref);
+    }
+  }
+
+  Wasm(WasmSourceType wasmSrc, std::string hash = std::string())
+      : src(std::move(wasmSrc)), _hash(std::move(hash)) {}
 
   // Create Wasm pointing to a path
   static Wasm path(std::string s, std::string hash = std::string());
@@ -63,7 +118,7 @@ class Manifest {
   std::map<std::string, std::string> allowedPaths;
   std::optional<uint64_t> timeout;
 
-  Manifest() {}
+  Manifest(std::vector<Wasm> wasm = {}) : wasm(std::move(wasm)) {}
 
   // Create manifest with a single Wasm from a path
   static Manifest wasmPath(std::string s, std::string hash = std::string());
@@ -76,7 +131,10 @@ class Manifest {
                             std::string hash = std::string());
   static Manifest wasmBytes(const std::vector<uint8_t> &data, std::string hash);
 
-  std::string json() const;
+  std::string json(const bool selfContained = true) const;
+
+  // Add Wasm
+  void addWasm(Wasm wasm);
 
   // Add Wasm from path
   void addWasmPath(std::string s, std::string hash = std::string());

src/manifest.cpp

@@ -31,47 +31,55 @@ static std::string base64_encode(const uint8_t *data, size_t len) {
 
 // Create Wasm pointing to a path
 Wasm Wasm::path(std::string s, std::string hash) {
-  return Wasm(WasmSourcePath, std::move(s), std::move(hash));
+  return Wasm(std::filesystem::path(std::move(s)), std::move(hash));
 }
 
 // Create Wasm pointing to a URL
 Wasm Wasm::url(std::string s, std::string hash, std::string method,
                std::map<std::string, std::string> headers) {
-  auto wasm = Wasm(WasmSourceURL, std::move(s), std::move(hash));
-  wasm.httpMethod = std::move(method);
-  wasm.httpHeaders = std::move(headers);
-  return wasm;
+  return Wasm(WasmURL(std::move(s), std::move(method), std::move(headers)),
+              std::move(hash));
 }
 
 // Create Wasm from bytes of a module
 Wasm Wasm::bytes(const uint8_t *data, const size_t len, std::string hash) {
-  std::string s = base64_encode(data, len);
-  return Wasm(WasmSourceBytes, std::move(s), std::move(hash));
+  return Wasm(WasmBytes(data, len), std::move(hash));
 }
 
 Wasm Wasm::bytes(const std::vector<uint8_t> &data, std::string hash) {
-  return Wasm::bytes(data.data(), data.size(), hash);
+  return Wasm::bytes(data.data(), data.size(), std::move(hash));
 }
 
 class Serializer {
 public:
-  static Json::Value json(const Wasm &wasm) {
+  static Json::Value json(const Wasm &wasm, const bool selfContained = true) {
     Json::Value doc;
 
-    if (wasm.source == WasmSourcePath) {
-      doc["path"] = wasm.ref;
-    } else if (wasm.source == WasmSourceURL) {
-      doc["url"] = wasm.ref;
-      doc["method"] = wasm.httpMethod;
-      if (!wasm.httpHeaders.empty()) {
+    if (std::holds_alternative<std::filesystem::path>(wasm.src)) {
+      doc["path"] = std::string(std::get<std::filesystem::path>(wasm.src));
+    } else if (std::holds_alternative<WasmURL>(wasm.src)) {
+      const auto &wasmURL = std::get<WasmURL>(wasm.src);
+      doc["url"] = wasmURL.url;
+      doc["method"] = wasmURL.httpMethod;
+      if (!wasmURL.httpHeaders.empty()) {
         Json::Value h;
-        for (auto k : wasm.httpHeaders) {
+        for (auto k : wasmURL.httpHeaders) {
           h[k.first] = k.second;
         }
         doc["headers"] = h;
       }
-    } else if (wasm.source == WasmSourceBytes) {
-      doc["data"] = wasm.ref;
+    } else if (std::holds_alternative<WasmBytes>(wasm.src)) {
+      const auto &wasmBytes = std::get<WasmBytes>(wasm.src);
+      auto src = wasmBytes.get();
+      auto srcSize = wasmBytes.getSize();
+      if (selfContained) {
+        doc["data"] = base64_encode(src, srcSize);
+      } else {
+        Json::Value data;
+        data["ptr"] = reinterpret_cast<uint64_t>(src);
+        data["len"] = static_cast<uint64_t>(srcSize);
+        doc["data"] = data;
+      }
     }
 
     if (!wasm._hash.empty()) {
@@ -82,11 +90,11 @@ class Serializer {
   }
 };
 
-std::string Manifest::json() const {
+std::string Manifest::json(const bool selfContained) const {
   Json::Value doc;
   Json::Value wasm;
-  for (auto w : this->wasm) {
-    wasm.append(Serializer::json(w));
+  for (const auto &w : this->wasm) {
+    wasm.append(Serializer::json(w, selfContained));
   }
 
   doc["wasm"] = wasm;
@@ -126,33 +134,28 @@ std::string Manifest::json() const {
 }
 
 Manifest Manifest::wasmPath(std::string s, std::string hash) {
-  Manifest m;
-  m.addWasmPath(std::move(s), std::move(hash));
-  return m;
+  return Manifest({Wasm(std::filesystem::path(std::move(s)), std::move(hash))});
 }
 
 // Create manifest with a single Wasm from a URL
 Manifest Manifest::wasmURL(std::string s, std::string hash) {
-  Manifest m;
-  m.addWasmURL(std::move(s), std::move(hash));
-  return m;
+  return Manifest({Wasm(WasmURL(std::move(s)), std::move(hash))});
 }
 
 // Create manifest from Wasm data
 Manifest Manifest::wasmBytes(const uint8_t *data, const size_t len,
                              std::string hash) {
-  Manifest m;
-  m.addWasmBytes(data, len, std::move(hash));
-  return m;
+  return Manifest({Wasm(WasmBytes(data, len), std::move(hash))});
 }
 
 Manifest Manifest::wasmBytes(const std::vector<uint8_t> &data,
                              std::string hash) {
-  Manifest m;
-  m.addWasmBytes(data, std::move(hash));
-  return m;
+  return Manifest::wasmBytes(data.data(), data.size(), std::move(hash));
 }
 
+// Add Wasm
+void Manifest::addWasm(Wasm wasm) { this->wasm.push_back(std::move(wasm)); }
+
 // Add Wasm from path
 void Manifest::addWasmPath(std::string s, std::string hash) {
   Wasm w = Wasm::path(std::move(s), std::move(hash));

src/plugin.cpp

@@ -41,7 +41,7 @@ Plugin::CancelHandle Plugin::cancelHandle() {
 // Create a new plugin from Manifest
 Plugin::Plugin(const Manifest &manifest, bool withWasi,
                std::vector<Function> functions)
-    : Plugin(manifest.json(), withWasi, std::move(functions)) {}
+    : Plugin(manifest.json(false), withWasi, std::move(functions)) {}
 
 bool Plugin::CancelHandle::cancel() {
   return extism_plugin_cancel(this->handle);