Merge pull request #476 from streamingfast/feature/allow-wasmbindgen-…

…shims

This enables running module with wasmbindgen shims by specifying runtime extension `wasm-bindgen-shims`

docs/release-notes/change-log.md

@@ -11,9 +11,46 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## v1.7.1 (Unreleased)
 
-* Substreams clients now enable gzip compression over the network (already supported by servers)
-* Fixed a failure in protogen where duplicate files would "appear multiple times" and fail
-* Fixed bug with block rate underflow in `gui`
+### Highlights
+
+- Substreams engine is now able run Rust code that depends on `solana_program` in Solana land to decode and `alloy/ether-rs` in Ethereum land
+
+#### How to use `solana_program` or `alloy`/`ether-rs`
+
+Those libraries when used in a `wasm32-unknown-unknown` context creates in a bunch of [wasmbindgen](https://rustwasm.github.io/wasm-bindgen/) imports in the resulting Substreams Rust code, imports that led to runtime errors because Substreams engine didn't know about those special imports until today.
+
+The Substreams engine is now able to "shims" those `wasmbindgen` imports enabling you to run code that depends libraries like `solana_program` and `alloy/ether-rs` which are known to pull those `wasmbindgen` imports. This is going to work as long as you do not actually call those special imports. Normal usage of those libraries don't accidentally call those methods normally. If they are called, the WASM module will fail at runtime and stall the Substreams module from going forward.
+
+To enable this feature, you need to explicitly opt-in by appending a `+wasm-bindgen-shims` at the end of the binary's type in your Substreams manifest:
+
+```yaml
+binaries:
+  default:
+    type: wasm/rust-v1
+    file: <some_file>
+```
+
+to become
+
+```yaml
+binaries:
+  default:
+    type: wasm/rust-v1+wasm-bindgen-shims
+    file: <some_file>
+```
+
+### Others
+
+* Substreams clients now enable gzip compression over the network (already supported by servers).
+
+* Substreams binary type can now be optionally composed of runtime extensions by appending a `+<extension>,[<extesions...>]` at the end of the binary type. Extensions are `key[=value]` that are runtime specifics.
+
+  > [!NOTE]
+  > If you were a library author and parsing generic Substreams manifest(s), you will now need to handle that possibility in the binary type. If you were reading the field without any processing, you don't have to change nothing.
+
+* Fixed a failure in protogen where duplicate files would "appear multiple times" and fail.
+
+* Fixed bug with block rate underflow in `gui`.
 
 ## v1.7.0
 

manifest/binary.go

@@ -0,0 +1,24 @@
+package manifest
+
+import "strings"
+
+// SplitBinaryType splits a binary type in two components: the type ID and the raw extensions.
+//
+// The format is `<id>+<extensions>`, where `<id>` is the binary type ID and `<extensions>`
+// is a comma-separated list of runtime extensions that are type specific. The `+<extensions>` part
+// is optional and can be omitted.
+//
+// So we accept the following formats:
+//
+//   - wasm/rust-v1
+//   - wasm/rust-v1+wasm-bindgen-shims
+//   - wasm/rust-v1+wasm-bindgen-shims,other-extension=value
+//
+// This method returns the ID and the raw unsplitted extensions string.
+// The input `wasm/rust-v1+wasm-bindgen-shims,other-extension=value` would
+// result in ("wasm/rust-v1", "wasm-bindgen-shims,other-extension=value") being
+// returned to your
+func SplitBinaryType(in string) (typeID string, rawExtensions string) {
+	typeID, rawExtensions, _ = strings.Cut(in, "+")
+	return
+}

manifest/package.go

@@ -363,7 +363,12 @@ func (r *manifestConverter) convertToPkg(m *Manifest) (pkg *pbsubstreams.Package
 			return nil, fmt.Errorf("module %q refers to %sbinary %q, which is not defined in the 'binaries' section of the manifest", mod.Name, implicit, binaryName)
 		}
 
-		switch binaryDef.Type {
+		wasmCodeTypeID, _ := SplitBinaryType(binaryDef.Type)
+		if err != nil {
+			return nil, fmt.Errorf("module %q: invalid code type %q: %w", mod.Name, binaryDef.Type, err)
+		}
+
+		switch wasmCodeTypeID {
 		case "wasm/rust-v1", "wasip1/tinygo-v1":
 			// OPTIM(abourget): also check if it's not already in
 			// `Binaries`, by comparing its, length + hash or value.

service/validate.go

@@ -7,6 +7,7 @@ import (
 	pbssinternal "github.com/streamingfast/substreams/pb/sf/substreams/intern/v2"
 	pbsubstreamsrpc "github.com/streamingfast/substreams/pb/sf/substreams/rpc/v2"
 	pbsubstreams "github.com/streamingfast/substreams/pb/sf/substreams/v1"
+	"github.com/streamingfast/substreams/wasm"
 )
 
 // Deprecated: use ValidateTier1Request
@@ -85,11 +86,16 @@ func validateModuleGraph(mods []*pbsubstreams.Module, outputModule string, block
 
 func validateBinaryTypes(bins []*pbsubstreams.Binary) error {
 	for _, binary := range bins {
-		switch binary.Type {
+		wasmCodeTypeID, _, err := wasm.ParseWASMCodeType(binary.Type)
+		if err != nil {
+			return fmt.Errorf("parse wasm type %q: %w", binary.Type, err)
+		}
+
+		switch wasmCodeTypeID {
 		case "wasm/rust-v1":
 		case "wasip1/tinygo-v1":
 		default:
-			return fmt.Errorf(`unsupported binary type: %q, please use "wasm/rust-v1" or "wasip1/tinygo-v1"`, binary.Type)
+			return fmt.Errorf(`unsupported binary type: %q, please use "wasm/rust-v1" or "wasip1/tinygo-v1"`, wasmCodeTypeID)
 		}
 	}
 	return nil

test/integration_test.go

@@ -364,6 +364,23 @@ func Test_SimpleMapModule(t *testing.T) {
 	require.NoError(t, run.Run(t, "test_map"))
 }
 
+func Test_WASMBindgenShims(t *testing.T) {
+	run := newTestRun(t, 12, 14, 14, "map_block", "./testdata/wasmbindgen_substreams/wasmbindgen-substreams-v0.1.0.spkg")
+	run.NewBlockGenerator = func(startBlock uint64, inclusiveStopBlock uint64) TestBlockGenerator {
+		return &LinearBlockGenerator{
+			startBlock:         startBlock,
+			inclusiveStopBlock: inclusiveStopBlock + 10,
+		}
+	}
+	run.ParallelSubrequests = 1
+
+	require.NoError(t, run.Run(t, "test_wasmbindgenshims"))
+
+	mapOutput := run.MapOutput("map_block")
+	fmt.Println(mapOutput)
+
+}
+
 func Test_Early(t *testing.T) {
 	run := newTestRun(t, 12, 14, 14, "test_map", "./testdata/simple_substreams/substreams-test-v0.1.0.spkg")
 	run.Params = map[string]string{"test_map": "my test params"}