feat(store): use original custom FS header stores when FS store is en…

…abled PE-5196

Use the original custom (not KV based) FS header stores when the FS
header store is enabled. This is to preserve directory fan out and
symlinking behavior. The new KV FS store is also retained, but will
primarily be used to support testing.

src/lib/kvstore.ts → src/init/header-stores.ts

@@ -22,8 +22,12 @@ import { FsKVStore } from '../store/fs-kv-store.js';
 import { LmdbKVStore } from '../store/lmdb-kv-store.js';
 import { RedisKvStore } from '../store/redis-kv-store.js';
 import { KVBufferStore } from '../types.js';
+import { KvBlockStore } from '../store/kv-block-store.js';
+import { KvTransactionStore } from '../store/kv-transaction-store.js';
+import { FsBlockStore } from '../store/fs-block-store.js';
+import { FsTransactionStore } from '../store/fs-transaction-store.js';
 
-export const getKvBufferStore = ({
+const createKvBufferStore = ({
   pathKey,
   type,
   log,
@@ -53,9 +57,62 @@ export const getKvBufferStore = ({
       });
     }
 
-    // TODO: implement redis
     default: {
       throw new Error(`Invalid chain cache type: ${type}`);
     }
   }
 };
+
+export const makeBlockStore = ({
+  log,
+  type,
+}: {
+  log: winston.Logger;
+  type: string;
+}) => {
+  const pathKey = 'partial-blocks';
+  if (type === 'fs') {
+    log.info('Using FsBlockStore');
+    return new FsBlockStore({
+      log,
+      baseDir: `data/headers/${pathKey}`,
+      tmpDir: `data/tmp/${pathKey}`,
+    });
+  } else {
+    return new KvBlockStore({
+      log,
+      kvBufferStore: createKvBufferStore({
+        log,
+        pathKey,
+        type: config.CHAIN_CACHE_TYPE,
+      }),
+    });
+  }
+};
+
+export const makeTxStore = ({
+  log,
+  type,
+}: {
+  log: winston.Logger;
+  type: string;
+}) => {
+  const pathKey = 'partial-txs';
+  if (type === 'fs') {
+    log.info('Using FsTransactionStore');
+    return new FsTransactionStore({
+      log,
+      baseDir: `data/headers/${pathKey}`,
+      tmpDir: `data/tmp/${pathKey}`,
+    });
+  } else {
+    return new KvTransactionStore({
+      log,
+      kvBufferStore: createKvBufferStore({
+        log,
+        pathKey,
+        type: config.CHAIN_CACHE_TYPE,
+      }),
+    });
+  }
+};

src/system.ts

@@ -29,7 +29,7 @@ import { StandaloneSqliteDatabase } from './database/standalone-sqlite.js';
 import * as events from './events.js';
 import { MatchTags } from './filters.js';
 import { UniformFailureSimulator } from './lib/chaos.js';
-import { getKvBufferStore } from './lib/kvstore.js';
+import { makeBlockStore, makeTxStore } from './init/header-stores.js';
 import { currentUnixTimestamp } from './lib/time.js';
 import log from './log.js';
 import * as metrics from './metrics.js';
@@ -38,8 +38,6 @@ import { StreamingManifestPathResolver } from './resolution/streaming-manifest-p
 import { TrustedGatewayArNSResolver } from './resolution/trusted-gateway-arns-resolver.js';
 import { FsChunkDataStore } from './store/fs-chunk-data-store.js';
 import { FsDataStore } from './store/fs-data-store.js';
-import { KvBlockStore } from './store/kv-block-store.js';
-import { KvTransactionStore } from './store/kv-transaction-store.js';
 import {
   BlockListValidator,
   BundleIndex,
@@ -73,21 +71,13 @@ export const arweaveClient = new ArweaveCompositeClient({
   arweave,
   trustedNodeUrl: config.TRUSTED_NODE_URL,
   skipCache: config.SKIP_CACHE,
-  blockStore: new KvBlockStore({
+  blockStore: makeBlockStore({
     log,
-    kvBufferStore: getKvBufferStore({
-      log,
-      pathKey: 'partial-blocks',
-      type: config.CHAIN_CACHE_TYPE,
-    }),
+    type: config.CHAIN_CACHE_TYPE,
   }),
-  txStore: new KvTransactionStore({
+  txStore: makeTxStore({
     log,
-    kvBufferStore: getKvBufferStore({
-      log,
-      pathKey: 'partial-txs',
-      type: config.CHAIN_CACHE_TYPE,
-    }),
+    type: config.CHAIN_CACHE_TYPE,
   }),
   failureSimulator: new UniformFailureSimulator({
     failureRate: config.SIMULATED_REQUEST_FAILURE_RATE,