Simplifying caching without tmp files

apify · Jul 28, 2023 · 3aabcc8 · 3aabcc8
1 parent 1a4872f
commit 3aabcc8
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 9 deletions.
diff --git a/templates/js-langchain/package.json b/templates/js-langchain/package.json
@@ -10,8 +10,7 @@
         "apify": "^3.0.0",
         "hnswlib-node": "^1.4.2",
         "langchain": "^0.0.82",
-        "tar": "^6.1.14",
-        "temp-write": "^5.0.0"
+        "tar": "^6.1.14"
     },
     "scripts": {
         "start": "node src/main.js",

diff --git a/templates/js-langchain/src/vector_index_cache.js b/templates/js-langchain/src/vector_index_cache.js
@@ -1,7 +1,7 @@
 import { Actor } from 'apify';
 import { createHash } from 'crypto';
-import { readFile } from 'node:fs/promises';
-import tempWrite from 'temp-write';
+import { finished } from 'node:stream/promises';
+import { Readable } from 'node:stream';
 import tar from 'tar';
 
 const VECTOR_INDEX_CACHE_STORE_NAME = 'vector-index-cache';
@@ -26,10 +26,9 @@ function getIndexCacheKey(config) {
  */
 export async function cacheVectorIndex(config, indexPath) {
     const vectorIndexCacheStore = await Actor.openKeyValueStore(VECTOR_INDEX_CACHE_STORE_NAME);
-    const tempFilePath = await tempWrite(tar.c({}, [indexPath]));
-    const packedVectorIndex = await readFile(tempFilePath);
+    const packedVectorIndexStream = tar.c({}, [indexPath]);
 
-    await vectorIndexCacheStore.setValue(getIndexCacheKey(config), packedVectorIndex, { contentType: 'application/tar' });
+    await vectorIndexCacheStore.setValue(getIndexCacheKey(config), packedVectorIndexStream, { contentType: 'application/tar' });
 }
 
 /**
@@ -44,8 +43,7 @@ export async function retrieveVectorIndex(config) {
     const vectorIndexRecord = await vectorIndexCacheStore.getValue(getIndexCacheKey(config));
     if (!vectorIndexRecord) return false;
 
-    const tempFilePath = await tempWrite(vectorIndexRecord);
-    await tar.x({ file: tempFilePath, strip: 1, C: '.' });
+    await finished(Readable.from(vectorIndexRecord).pipe(tar.x({ strip: 1, C: '.' })));
 
     return true;
 }