Benchmarks for different variable parsing techniques

package.json

@@ -74,6 +74,7 @@
     "@types/lodash.memoize": "^4.1.7",
     "@types/lodash.merge": "^4.6.7",
     "@types/lodash.mergewith": "^4.6.7",
+    "@types/minimist": "^1.2.2",
     "@types/node": "^20.6.0",
     "@typescript-eslint/eslint-plugin": "6.7.0",
     "@typescript-eslint/parser": "6.7.0",
@@ -84,6 +85,7 @@
     "graphql": "^16.8.0",
     "jest": "^29.7.0",
     "lint-staged": "^14.0.1",
+    "minimist": "^1.2.8",
     "prettier": "^3.0.3",
     "typescript": "^5.2.2"
   },

src/__benchmarks__/benchmarks.ts

@@ -1,14 +1,20 @@
 #!/usr/bin/env node -r @swc-node/register
 
 import Benchmark from "benchmark";
+import minimist from "minimist";
 import {
   DocumentNode,
   execute,
   getIntrospectionQuery,
   GraphQLSchema,
   parse
 } from "graphql";
-import { compileQuery, isCompiledQuery, isPromise } from "../execution";
+import {
+  compileQuery,
+  CompilerOptions,
+  isCompiledQuery,
+  isPromise
+} from "../execution";
 import {
   query as fewResolversQuery,
   schema as fewResolversSchema
@@ -21,11 +27,17 @@ import {
   query as nestedArrayQuery,
   schema as nestedArraySchema
 } from "./schema-nested-array";
+import {
+  query as variablesShallowQuery,
+  schema as variablesShallowSchema,
+  variables as variablesShallowVariables
+} from "./variables-parsing-shallow";
 
 interface BenchmarkMaterial {
   query: DocumentNode;
   schema: GraphQLSchema;
   variables?: any;
+  options?: Partial<CompilerOptions>;
 }
 
 const benchmarks: { [key: string]: BenchmarkMaterial } = {
@@ -47,16 +59,72 @@ const benchmarks: { [key: string]: BenchmarkMaterial } = {
     schema: nestedArraySchema(),
     query: nestedArrayQuery,
     variables: { id: "2", width: 300, height: 500 }
+  },
+  variablesWithNewJit: {
+    schema: variablesShallowSchema(),
+    query: variablesShallowQuery,
+    variables: variablesShallowVariables,
+    options: {
+      variableParser: "jit-new"
+    }
+  },
+  variablesWithOldJit: {
+    schema: variablesShallowSchema(false),
+    query: variablesShallowQuery,
+    variables: variablesShallowVariables,
+    options: {
+      variableParser: "jit-old"
+    }
+  },
+  variablesWithGraphQLJS: {
+    schema: variablesShallowSchema(),
+    query: variablesShallowQuery,
+    variables: variablesShallowVariables,
+    options: {
+      variableParser: "graphql-js"
+    }
   }
 };
 
-async function runBenchmarks() {
-  const skipJS = process.argv[2] === "skip-js";
-  const skipJSON = process.argv[2] === "skip-json";
+async function runBenchmarks(argv: string[]) {
+  const args = minimist(argv);
+  const help = args["help"];
+
+  const availableBenchmarks = Object.entries(benchmarks);
+
+  if (help) {
+    console.log(
+      `
+Usage: yarn benchmark [options]
+
+Options:
+  --skip-js    Skip graphql-js benchmarks
+  --skip-json  Skip JSON.stringify benchmarks
+  --help       Show this help
+  --bench      Run only the specified benchmarks (comma separated)
+
+Available benchmarks:
+${availableBenchmarks.map(([bench]) => `  - ${bench}`).join("\n")}
+`.trim()
+    );
+    return;
+  }
+
+  const skipJS = args["skip-js"];
+  const skipJSON = args["skip-json"];
+  const benchsToRunArg = args["bench"];
+  const benchmarksToRun =
+    benchsToRunArg && benchsToRunArg.split(",").filter((b: string) => b);
+
+  const filteredBenchmarks = benchmarksToRun
+    ? availableBenchmarks.filter(([bench]) => benchmarksToRun.includes(bench))
+    : availableBenchmarks;
+
   const benchs = await Promise.all(
-    Object.entries(benchmarks).map(
-      async ([bench, { query, schema, variables }]) => {
+    filteredBenchmarks.map(
+      async ([bench, { query, schema, variables, options }]) => {
         const compiledQuery = compileQuery(schema, query, undefined, {
+          ...options,
           debug: true
         } as any);
         if (!isCompiledQuery(compiledQuery)) {
@@ -71,6 +139,11 @@ async function runBenchmarks() {
               .__DO_NOT_USE_THIS_OR_YOU_WILL_BE_FIRED_compilation.length
           }`
         );
+        console.log(
+          `size of function for variableCompilation ${bench}: ${(
+            compiledQuery as any
+          ).__DO_NOT_USE_THIS_OR_YOU_WILL_BE_FIRED_variableCompilation?.length}`
+        );
         const graphqlJsResult = await execute({
           schema,
           document: query,
@@ -168,7 +241,7 @@ async function runBenchmarks() {
 }
 
 // eslint-disable-next-line
-runBenchmarks().catch(console.error);
+runBenchmarks(process.argv.slice(2)).catch(console.error);
 
 function isNotNull<T>(a: T | null | undefined): a is T {
   return a != null;

src/__benchmarks__/variables-parsing-shallow.ts

@@ -0,0 +1,172 @@
+import { makeExecutableSchema } from "@graphql-tools/schema";
+import { parse } from "graphql";
+
+const typeDefs = `
+type Query {
+  products(filter: Filter): [Product]
+}
+input Filter {
+  and: AndFilter
+  or: OrFilter
+  like: String
+}
+input AndFilter {
+  left: Filter
+  right: Filter
+}
+input OrFilter {
+  left: Filter
+  right: Filter
+}
+type Product {
+  id: ID!
+  name: String!
+}
+`;
+
+const typeDefsOld = `
+type Query {
+  products(filter: Filter): [Product]
+}
+input Filter {
+  and: AndFilter
+  or: OrFilter
+  like: String
+}
+input AndFilter {
+  left: L2Filter
+  right: L2Filter
+}
+input OrFilter {
+  left: L2Filter
+  right: L2Filter
+}
+input L2Filter {
+  and: L2AndFilter
+  or: L2OrFilter
+  like: String
+}
+input L2AndFilter {
+  left: L3Filter
+  right: L3Filter
+}
+input L2OrFilter {
+  left: L3Filter
+  right: L3Filter
+}
+input L3Filter {
+  like: String
+}
+type Product {
+  id: ID!
+  name: String!
+}
+`;
+
+export function schema(withRecursion = true) {
+  const schema = makeExecutableSchema({
+    typeDefs: withRecursion ? typeDefs : typeDefsOld,
+    resolvers: {
+      Query: {
+        async products(_, { filter }) {
+          return products.filter((product) =>
+            productSatisfiesFilter(product, filter)
+          );
+        }
+      }
+    }
+  });
+
+  return schema;
+}
+
+export const query = parse(`
+query ($filter1: Filter) {
+  products(filter: $filter1) {
+    id
+    name
+  }
+}
+`);
+
+export const variables = {
+  filter1: {
+    and: {
+      left: {
+        like: "Chrome"
+      },
+      right: {
+        or: {
+          left: {
+            like: "FreeBSD"
+          },
+          right: {
+            like: "Samsung"
+          }
+        }
+      }
+    }
+  }
+};
+
+function productSatisfiesFilter(
+  product: (typeof products)[0],
+  filter: any
+): boolean {
+  if (filter.and) {
+    return (
+      productSatisfiesFilter(product, filter.and.left) &&
+      productSatisfiesFilter(product, filter.and.right)
+    );
+  } else if (filter.or) {
+    return (
+      productSatisfiesFilter(product, filter.or.left) ||
+      productSatisfiesFilter(product, filter.or.right)
+    );
+  } else {
+    return product.name.includes(filter.like);
+  }
+}
+
+const products = [
+  {
+    id: "1",
+    name: "Mozilla - Android - SM-G960F - AppleWebKit - Chrome - Mobile Safari - Linux - Samsung Galaxy S9"
+  },
+  {
+    id: "2",
+    name: "Mozilla - Linux - Ubuntu - Gecko - Firefox - - Linux - Desktop"
+  },
+  {
+    id: "3",
+    name: "Mozilla - Mac - Intel Mac OS X 10_15_7 - AppleWebKit - Safari - - Mac - Desktop"
+  },
+  {
+    id: "4",
+    name: "Mozilla - Windows - Windows NT 10.0 - AppleWebKit - Chrome - Safari - Windows - Desktop"
+  },
+  {
+    id: "5",
+    name: "Mozilla - Linux - - AppleWebKit - Chrome - Safari - Linux - Desktop"
+  },
+  {
+    id: "6",
+    name: "Mozilla - Chrome OS - CrOS x86_64 13904.93.0 - AppleWebKit - Chrome - Safari - Chrome OS - Desktop"
+  },
+  {
+    id: "7",
+    name: "Mozilla - FreeBSD - FreeBSD amd64 - Gecko - Firefox - - FreeBSD - Desktop"
+  },
+  {
+    id: "8",
+    name: "Mozilla - Android - SM-G960F - AppleWebKit - Chrome - Mobile Safari - Linux - Samsung Galaxy S9"
+  },
+  {
+    id: "9",
+    name: "Mozilla - iOS - iPhone - AppleWebKit - Safari - - iOS - iPhone"
+  },
+  {
+    id: "10",
+    name: "Mozilla - Linux - Ubuntu - Gecko - Firefox - - Linux - Desktop"
+  }
+];