Merge pull request #176 from autonomys/fix/chunker-issue

fix: file chunking due to mix of nodeSize and chunkSize limitation

packages/auto-dag-data/src/ipld/chunker.ts

@@ -15,7 +15,7 @@ type ChunkerLimits = {
 
 type ChunkerOptions = ChunkerLimits & FileUploadOptions
 
-const DEFAULT_NODE_MAX_SIZE = 65535
+export const DEFAULT_NODE_MAX_SIZE = 65535
 
 // u8 -> 1 byte (may grow in the future but unlikely further than 255)
 const NODE_TYPE_SIZE = 1
@@ -51,12 +51,12 @@ export const processFileToIPLDFormat = (
   totalSize: bigint,
   filename?: string,
   {
-    maxNodeSize = DEFAULT_MAX_CHUNK_SIZE,
+    maxNodeSize = DEFAULT_NODE_MAX_SIZE,
     maxLinkPerNode = DEFAULT_MAX_LINK_PER_NODE,
     encryption = undefined,
     compression = undefined,
   }: Partial<ChunkerOptions> = {
-    maxNodeSize: DEFAULT_MAX_CHUNK_SIZE,
+    maxNodeSize: DEFAULT_NODE_MAX_SIZE,
     maxLinkPerNode: DEFAULT_MAX_LINK_PER_NODE,
     encryption: undefined,
     compression: undefined,
@@ -78,7 +78,7 @@ export const processMetadataToIPLDFormat = async (
   blockstore: BaseBlockstore,
   metadata: OffchainMetadata,
   limits: { maxNodeSize: number; maxLinkPerNode: number } = {
-    maxNodeSize: DEFAULT_MAX_CHUNK_SIZE,
+    maxNodeSize: DEFAULT_NODE_MAX_SIZE,
     maxLinkPerNode: DEFAULT_MAX_LINK_PER_NODE,
   },
 ): Promise<CID> => {
@@ -107,12 +107,12 @@ const processBufferToIPLDFormat = async (
   totalSize: bigint,
   builders: Builders,
   {
-    maxNodeSize: maxNodeSize = DEFAULT_MAX_CHUNK_SIZE,
+    maxNodeSize: maxNodeSize = DEFAULT_NODE_MAX_SIZE,
     maxLinkPerNode = DEFAULT_MAX_LINK_PER_NODE,
     encryption = undefined,
     compression = undefined,
   }: ChunkerOptions = {
-    maxNodeSize: DEFAULT_MAX_CHUNK_SIZE,
+    maxNodeSize: DEFAULT_NODE_MAX_SIZE,
     maxLinkPerNode: DEFAULT_MAX_LINK_PER_NODE,
     encryption: undefined,
     compression: undefined,
@@ -147,12 +147,12 @@ export const processBufferToIPLDFormatFromChunks = async (
   totalSize: bigint,
   builders: Builders,
   {
-    maxNodeSize: maxNodeSize = DEFAULT_MAX_CHUNK_SIZE,
+    maxNodeSize: maxNodeSize = DEFAULT_NODE_MAX_SIZE,
     maxLinkPerNode = DEFAULT_MAX_LINK_PER_NODE,
     encryption = undefined,
     compression = undefined,
   }: Partial<ChunkerOptions> = {
-    maxNodeSize: DEFAULT_MAX_CHUNK_SIZE,
+    maxNodeSize: DEFAULT_NODE_MAX_SIZE,
     maxLinkPerNode: DEFAULT_MAX_LINK_PER_NODE,
     encryption: undefined,
     compression: undefined,
@@ -214,12 +214,12 @@ export const processFolderToIPLDFormat = async (
   size: bigint,
   {
     maxLinkPerNode = DEFAULT_MAX_LINK_PER_NODE,
-    maxNodeSize: maxNodeSize = DEFAULT_MAX_CHUNK_SIZE,
+    maxNodeSize: maxNodeSize = DEFAULT_NODE_MAX_SIZE,
     compression = undefined,
     encryption = undefined,
   }: Partial<ChunkerOptions> = {
     maxLinkPerNode: DEFAULT_MAX_LINK_PER_NODE,
-    maxNodeSize: DEFAULT_MAX_CHUNK_SIZE,
+    maxNodeSize: DEFAULT_NODE_MAX_SIZE,
     compression: undefined,
     encryption: undefined,
   },
@@ -261,15 +261,17 @@ export const processChunksToIPLDFormat = async (
   blockstore: BaseBlockstore,
   chunks: AwaitIterable<Buffer>,
   builders: Builders,
-  { maxNodeSize = DEFAULT_MAX_CHUNK_SIZE }: { maxNodeSize?: number },
+  { maxChunkSize = DEFAULT_MAX_CHUNK_SIZE }: { maxChunkSize?: number } = {
+    maxChunkSize: DEFAULT_MAX_CHUNK_SIZE,
+  },
 ): Promise<Buffer> => {
   const bufferChunks = chunkBuffer(chunks, {
-    maxChunkSize: maxNodeSize - NODE_METADATA_SIZE,
+    maxChunkSize,
     ignoreLastChunk: false,
   })
 
   for await (const chunk of bufferChunks) {
-    if (chunk.byteLength < maxNodeSize) {
+    if (chunk.byteLength < maxChunkSize) {
       return chunk
     }
 
@@ -283,7 +285,7 @@ export const processChunksToIPLDFormat = async (
 
 export const ensureNodeMaxSize = (
   node: PBNode,
-  maxSize: number = DEFAULT_MAX_CHUNK_SIZE,
+  maxSize: number = DEFAULT_NODE_MAX_SIZE,
 ): PBNode => {
   const nodeSize = encodeNode(node).byteLength
   if (nodeSize > maxSize) {

packages/auto-dag-data/src/ipld/nodes.ts

@@ -2,13 +2,13 @@ import { CID } from 'multiformats/cid'
 import { FileUploadOptions, OffchainMetadata } from '../metadata/index.js'
 import { encodeIPLDNodeData, MetadataType } from '../metadata/onchain/index.js'
 import { stringifyMetadata } from '../utils/metadata.js'
-import { DEFAULT_MAX_CHUNK_SIZE, ensureNodeMaxSize } from './chunker.js'
+import { DEFAULT_NODE_MAX_SIZE, ensureNodeMaxSize } from './chunker.js'
 import { createNode, PBNode } from './index.js'
 
 /// Creates a file chunk ipld node
 export const createFileChunkIpldNode = (
   data: Buffer,
-  maxNodeSize: number = DEFAULT_MAX_CHUNK_SIZE,
+  maxNodeSize: number = DEFAULT_NODE_MAX_SIZE,
 ): PBNode =>
   ensureNodeMaxSize(
     createNode(
@@ -31,7 +31,7 @@ export const createChunkedFileIpldNode = (
   size: bigint,
   linkDepth: number,
   name?: string,
-  maxNodeSize: number = DEFAULT_MAX_CHUNK_SIZE,
+  maxNodeSize: number = DEFAULT_NODE_MAX_SIZE,
   uploadOptions?: FileUploadOptions,
 ): PBNode =>
   ensureNodeMaxSize(
@@ -53,7 +53,7 @@ export const createFileInlinkIpldNode = (
   links: CID[],
   size: number,
   linkDepth: number,
-  maxNodeSize: number = DEFAULT_MAX_CHUNK_SIZE,
+  maxNodeSize: number = DEFAULT_NODE_MAX_SIZE,
 ): PBNode =>
   ensureNodeMaxSize(
     createNode(
@@ -74,7 +74,7 @@ export const createSingleFileIpldNode = (
   data: Buffer,
   name?: string,
   uploadOptions?: FileUploadOptions,
-  maxNodeSize: number = DEFAULT_MAX_CHUNK_SIZE,
+  maxNodeSize: number = DEFAULT_NODE_MAX_SIZE,
 ): PBNode =>
   ensureNodeMaxSize(
     createNode(
@@ -98,7 +98,7 @@ export const createMetadataInlinkIpldNode = (
   links: CID[],
   size: number,
   linkDepth: number,
-  maxNodeSize: number = DEFAULT_MAX_CHUNK_SIZE,
+  maxNodeSize: number = DEFAULT_NODE_MAX_SIZE,
 ): PBNode =>
   ensureNodeMaxSize(
     createNode(
@@ -129,7 +129,7 @@ export const createSingleMetadataIpldNode = (data: Buffer, name?: string): PBNod
 
 export const createMetadataChunkIpldNode = (
   data: Buffer,
-  maxNodeSize: number = DEFAULT_MAX_CHUNK_SIZE,
+  maxNodeSize: number = DEFAULT_NODE_MAX_SIZE,
 ): PBNode =>
   ensureNodeMaxSize(
     createNode(
@@ -148,7 +148,7 @@ export const createChunkedMetadataIpldNode = (
   size: bigint,
   linkDepth: number,
   name?: string,
-  maxNodeSize: number = DEFAULT_MAX_CHUNK_SIZE,
+  maxNodeSize: number = DEFAULT_NODE_MAX_SIZE,
 ): PBNode =>
   ensureNodeMaxSize(
     createNode(
@@ -171,7 +171,7 @@ export const createFolderIpldNode = (
   name: string,
   linkDepth: number,
   size: bigint,
-  maxNodeSize: number = DEFAULT_MAX_CHUNK_SIZE,
+  maxNodeSize: number = DEFAULT_NODE_MAX_SIZE,
   uploadOptions?: FileUploadOptions,
 ): PBNode =>
   ensureNodeMaxSize(
@@ -191,7 +191,7 @@ export const createFolderIpldNode = (
 export const createFolderInlinkIpldNode = (
   links: CID[],
   linkDepth: number,
-  maxNodeSize: number = DEFAULT_MAX_CHUNK_SIZE,
+  maxNodeSize: number = DEFAULT_NODE_MAX_SIZE,
 ): PBNode =>
   ensureNodeMaxSize(
     createNode(
@@ -207,7 +207,7 @@ export const createFolderInlinkIpldNode = (
 /// Creates a metadata ipld node
 export const createMetadataNode = (
   metadata: OffchainMetadata,
-  maxNodeSize: number = DEFAULT_MAX_CHUNK_SIZE,
+  maxNodeSize: number = DEFAULT_NODE_MAX_SIZE,
 ): PBNode => {
   const data = Buffer.from(stringifyMetadata(metadata))
 

packages/auto-dag-data/tests/nodes.spec.ts

@@ -4,7 +4,7 @@ import {
   createFileChunkIpldNode,
   createSingleFileIpldNode,
 } from '../src/index.js'
-import { createNode, DEFAULT_MAX_CHUNK_SIZE } from '../src/ipld/index.js'
+import { createNode, DEFAULT_MAX_CHUNK_SIZE, DEFAULT_NODE_MAX_SIZE } from '../src/ipld/index.js'
 import { IPLDNodeData, MetadataType } from '../src/metadata/onchain/protobuf/OnchainMetadata.js'
 
 describe('node creation', () => {
@@ -30,7 +30,7 @@ describe('node creation', () => {
     })
 
     it('single file root node | buffer too large', () => {
-      const maxNodeSize = DEFAULT_MAX_CHUNK_SIZE
+      const maxNodeSize = DEFAULT_NODE_MAX_SIZE
       const buffer = Buffer.from('h'.repeat(maxNodeSize))
       expect(() => createSingleFileIpldNode(buffer, 'test.txt')).toThrow()
     })