fix(graphene): fix breakage due to kvstore url encoding fixes

Fixes #727.
google · Feb 12, 2025 · f203e24 · f203e24
1 parent 35a4af7
commit f203e24
Show file tree

Hide file tree

Showing 4 changed files with 58 additions and 48 deletions.
diff --git a/src/datasource/graphene/backend.ts b/src/datasource/graphene/backend.ts
@@ -33,6 +33,7 @@ import {
   RENDER_RATIO_LIMIT,
   isBaseSegmentId,
   parseGrapheneError,
+  getHttpSource,
 } from "#src/datasource/graphene/base.js";
 import { decodeManifestChunk } from "#src/datasource/precomputed/backend.js";
 import { WithSharedKvStoreContextCounterpart } from "#src/kvstore/backend.js";
@@ -127,7 +128,8 @@ export class GrapheneMeshSource extends WithParameters(
   manifestRequestCount = new Map<string, number>();
   newSegments = new Uint64Set();
 
-  manifestKvStore = this.sharedKvStoreContext.kvStoreContext.getKvStore(
+  manifestHttpSource = getHttpSource(
+    this.sharedKvStoreContext.kvStoreContext,
     this.parameters.manifestUrl,
   );
   fragmentKvStore = this.sharedKvStoreContext.kvStoreContext.getKvStore(
@@ -148,14 +150,11 @@ export class GrapheneMeshSource extends WithParameters(
     if (isBaseSegmentId(chunk.objectId, parameters.nBitsForLayerId)) {
       return decodeManifestChunk(chunk, { fragments: [] });
     }
-    const { manifestKvStore } = this;
-    const manifestPath = `${manifestKvStore.path}/manifest/${chunk.objectId}:${parameters.lod}?verify=1&prepend_seg_ids=1`;
+    const { fetchOkImpl, baseUrl } = this.manifestHttpSource;
+    const manifestPath = `/manifest/${chunk.objectId}:${parameters.lod}?verify=1&prepend_seg_ids=1`;
     const response = await (
-      await readKvStore(manifestKvStore.store, manifestPath, {
-        throwIfMissing: true,
-        signal,
-      })
-    ).response.json();
+      await fetchOkImpl(baseUrl + manifestPath, { signal })
+    ).json();
     const chunkIdentifier = manifestPath;
     if (newSegments.has(chunk.objectId)) {
       const requestCount = (manifestRequestCount.get(chunkIdentifier) ?? 0) + 1;
@@ -251,7 +250,8 @@ export class GrapheneChunkedGraphChunkSource extends WithParameters(
   tempChunkDataSize: Uint32Array;
   tempChunkPosition: Float32Array;
 
-  kvStore = this.sharedKvStoreContext.kvStoreContext.getKvStore(
+  httpSource = getHttpSource(
+    this.sharedKvStoreContext.kvStoreContext,
     this.parameters.url,
   );
 
@@ -271,18 +271,16 @@ export class GrapheneChunkedGraphChunkSource extends WithParameters(
       `${chunkPosition[1]}-${chunkPosition[1] + chunkDataSize[1]}_` +
       `${chunkPosition[2]}-${chunkPosition[2] + chunkDataSize[2]}`;
 
-    const { kvStore } = this;
-
-    const request = readKvStore(
-      kvStore.store,
-      `${kvStore.path}/${chunk.segment}/leaves?int64_as_str=1&bounds=${bounds}`,
-      { signal, throwIfMissing: true },
+    const { fetchOkImpl, baseUrl } = this.httpSource;
+    const request = fetchOkImpl(
+      `${baseUrl}/${chunk.segment}/leaves?int64_as_str=1&bounds=${bounds}`,
+      { signal },
     );
     await this.withErrorMessage(
       request,
       `Fetching leaves of segment ${chunk.segment} in region ${bounds}: `,
     )
-      .then((res) => res.response.json())
+      .then((res) => res.json())
       .then((res) => {
         chunk.leaves = decodeChunkedGraphChunk(res.leaf_ids);
       })

diff --git a/src/datasource/graphene/base.ts b/src/datasource/graphene/base.ts
@@ -15,6 +15,9 @@
  */
 
 import type { ShardingParameters } from "#src/datasource/precomputed/base.js";
+import type { KvStoreContext } from "#src/kvstore/context.js";
+import { ReadableHttpKvStore } from "#src/kvstore/http/common.js";
+import { joinBaseUrlAndPath } from "#src/kvstore/url.js";
 import type {
   ChunkLayoutOptions,
   SliceViewChunkSource,
@@ -25,8 +28,7 @@ import type {
 } from "#src/sliceview/base.js";
 import { makeSliceViewChunkSpecification } from "#src/sliceview/base.js";
 import type { mat4 } from "#src/util/geom.js";
-import type { HttpError } from "#src/util/http_request.js";
-
+import type { FetchOk, HttpError } from "#src/util/http_request.js";
 import { Uint64 } from "#src/util/uint64.js";
 
 export const PYCG_APP_VERSION = 1;
@@ -151,3 +153,23 @@ export async function parseGrapheneError(e: HttpError) {
   }
   return undefined;
 }
+
+export interface HttpSource {
+  fetchOkImpl: FetchOk;
+  baseUrl: string;
+}
+
+export function getHttpSource(
+  kvStoreContext: KvStoreContext,
+  url: string,
+): HttpSource {
+  const { store, path } = kvStoreContext.getKvStore(url);
+  if (!(store instanceof ReadableHttpKvStore)) {
+    throw new Error(`Non-HTTP URL ${JSON.stringify(url)} not supported`);
+  }
+  const { fetchOkImpl, baseUrl } = store;
+  if (baseUrl.includes("?")) {
+    throw new Error(`Invalid URL ${baseUrl}: query parameters not supported`);
+  }
+  return { fetchOkImpl, baseUrl: joinBaseUrlAndPath(baseUrl, path) };
+}
diff --git a/src/datasource/graphene/frontend.ts b/src/datasource/graphene/frontend.ts
@@ -37,6 +37,7 @@ import { makeIdentityTransform } from "#src/coordinate_transform.js";
 import type {
   ChunkedGraphChunkSource as ChunkedGraphChunkSourceInterface,
   ChunkedGraphChunkSpecification,
+  HttpSource,
   MultiscaleMeshMetadata,
 } from "#src/datasource/graphene/base.js";
 import {
@@ -50,6 +51,7 @@ import {
   makeChunkedGraphChunkSpecification,
   MeshSourceParameters,
   PYCG_APP_VERSION,
+  getHttpSource,
 } from "#src/datasource/graphene/base.js";
 import type {
   DataSource,
@@ -71,7 +73,6 @@ import {
 } from "#src/datasource/precomputed/frontend.js";
 import { WithSharedKvStoreContext } from "#src/kvstore/chunk_source_frontend.js";
 import type { SharedKvStoreContext } from "#src/kvstore/frontend.js";
-import { ReadableHttpKvStore } from "#src/kvstore/http/common.js";
 import {
   ensureEmptyUrlSuffix,
   kvstoreEnsureDirectoryPipelineUrl,
@@ -165,7 +166,6 @@ import type { ValueOrError } from "#src/util/error.js";
 import { makeValueOrError, valueOrThrow } from "#src/util/error.js";
 import { EventActionMap } from "#src/util/event_action_map.js";
 import { mat4, vec3, vec4 } from "#src/util/geom.js";
-import type { FetchOk } from "#src/util/http_request.js";
 import { HttpError, isNotFoundError } from "#src/util/http_request.js";
 import {
   parseArray,
@@ -1578,27 +1578,23 @@ async function withErrorMessageHTTP<T>(
 export const GRAPH_SERVER_NOT_SPECIFIED = Symbol("Graph Server Not Specified.");
 
 class GrapheneGraphServerInterface {
-  private fetchOkImpl: FetchOk;
-  private url: string;
+  private httpSource: HttpSource;
   constructor(sharedKvStoreContext: SharedKvStoreContext, url: string) {
-    const { store, path } = sharedKvStoreContext.kvStoreContext.getKvStore(url);
-    if (store instanceof ReadableHttpKvStore) {
-      this.fetchOkImpl = store.fetchOkImpl;
-      this.url = store.baseUrl + path;
-    } else {
-      throw new Error(`Non-HTTP protocol not supported: ${url}`);
-    }
+    this.httpSource = getHttpSource(sharedKvStoreContext.kvStoreContext, url);
   }
 
   async getRoot(segment: Uint64, timestamp = "") {
     const timestampEpoch = new Date(timestamp).valueOf() / 1000;
 
-    const url = `${this.url}/node/${String(segment)}/root?int64_as_str=1${
-      Number.isNaN(timestampEpoch) ? "" : `&timestamp=${timestampEpoch}`
-    }`;
+    const { fetchOkImpl, baseUrl } = this.httpSource;
 
     const jsonResp = await withErrorMessageHTTP(
-      this.fetchOkImpl(url).then((response) => response.json()),
+      fetchOkImpl(
+        `${baseUrl}/node/${String(segment)}/root?int64_as_str=1${
+          Number.isNaN(timestampEpoch) ? "" : `&timestamp=${timestampEpoch}`
+        }`,
+        {},
+      ).then((response) => response.json()),
       {
         initialMessage: `Retrieving root for segment ${segment}`,
         errorPrefix: "Could not fetch root: ",
@@ -1612,12 +1608,8 @@ class GrapheneGraphServerInterface {
     second: SegmentSelection,
     annotationToNanometers: Float64Array,
   ): Promise<Uint64> {
-    const { url } = this;
-    if (url === "") {
-      return Promise.reject(GRAPH_SERVER_NOT_SPECIFIED);
-    }
-
-    const promise = this.fetchOkImpl(`${url}/merge?int64_as_str=1`, {
+    const { fetchOkImpl, baseUrl } = this.httpSource;
+    const promise = fetchOkImpl(`${baseUrl}/merge?int64_as_str=1`, {
       method: "POST",
       body: JSON.stringify([
         [
@@ -1649,12 +1641,8 @@ class GrapheneGraphServerInterface {
     second: SegmentSelection[],
     annotationToNanometers: Float64Array,
   ): Promise<Uint64[]> {
-    const { url } = this;
-    if (url === "") {
-      return Promise.reject(GRAPH_SERVER_NOT_SPECIFIED);
-    }
-
-    const promise = this.fetchOkImpl(`${url}/split?int64_as_str=1`, {
+    const { fetchOkImpl, baseUrl } = this.httpSource;
+    const promise = fetchOkImpl(`${baseUrl}/split?int64_as_str=1`, {
       method: "POST",
       body: JSON.stringify({
         sources: first.map((x) => [
@@ -1681,9 +1669,10 @@ class GrapheneGraphServerInterface {
   }
 
   async filterLatestRoots(segments: Uint64[]): Promise<Uint64[]> {
-    const url = `${this.url}/is_latest_roots`;
+    const { fetchOkImpl, baseUrl } = this.httpSource;
+    const url = `${baseUrl}/is_latest_roots`;
 
-    const promise = this.fetchOkImpl(url, {
+    const promise = fetchOkImpl(url, {
       method: "POST",
       body: JSON.stringify({
         node_ids: segments.map((x) => x.toJSON()),

diff --git a/src/datasource/state_share.ts b/src/datasource/state_share.ts
@@ -1,4 +1,5 @@
 import { ReadableHttpKvStore } from "#src/kvstore/http/common.js";
+import { joinBaseUrlAndPath } from "#src/kvstore/url.js";
 import { StatusMessage } from "#src/status.js";
 import { RefCounted } from "#src/util/disposable.js";
 import type { Viewer } from "#src/viewer.js";
@@ -87,7 +88,7 @@ export class StateShare extends RefCounted {
 
       StatusMessage.forPromise(
         store
-          .fetchOkImpl(store.baseUrl + path, {
+          .fetchOkImpl(joinBaseUrlAndPath(store.baseUrl, path), {
             method: "POST",
             headers: { "Content-Type": "application/json" },
             body: JSON.stringify(viewer.state.toJSON()),