Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(annotation): AnnotationLayerView now shows currently displayed annotations in MultiscaleAnnotationSource #504

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

feat(annotation): AnnotationLayerView now shows currently displayed annotations in MultiscaleAnnotationSource #504

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
54cfd89
feat(annotation): AnnotationLayerView now shows currently displayed a…
chrisj Sep 28, 2023
cf22bac
improved annotation list performance by only generating annotations t…
chrisj Jan 4, 2024
1f3f734
fix - serializedAnnotationsList wasn't being constructed for segmentF…
chrisj Jan 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 1 addition & 8 deletions src/neuroglancer/annotation/annotation_layer_state.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,14 +55,7 @@ export class WatchableAnnotationRelationshipStates extends
nestedContext.registerDisposer(segmentationGroupState.changed.add(this.changed.dispatch));
nestedContext.registerDisposer(registerNested((groupContext, groupState) => {
const {visibleSegments} = groupState;
let wasEmpty = visibleSegments.size === 0;
groupContext.registerDisposer(visibleSegments.changed.add(() => {
const isEmpty = visibleSegments.size === 0;
if (isEmpty !== wasEmpty) {
wasEmpty = isEmpty;
this.changed.dispatch();
}
}));
groupContext.registerDisposer(visibleSegments.changed.add(this.changed.dispatch));
}, segmentationGroupState));
}, segmentationState));
});
Expand Down
153 changes: 151 additions & 2 deletions src/neuroglancer/annotation/frontend_source.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,15 +15,20 @@
*/

import {Annotation, AnnotationId, AnnotationPropertySerializer, AnnotationPropertySpec, AnnotationReference, AnnotationSourceSignals, AnnotationType, annotationTypeHandlers, annotationTypes, fixAnnotationAfterStructuredCloning, makeAnnotationId, makeAnnotationPropertySerializers, SerializedAnnotations} from 'neuroglancer/annotation';
import {AnnotationLayerState} from 'neuroglancer/annotation/annotation_layer_state';
import {ANNOTATION_COMMIT_UPDATE_RESULT_RPC_ID, ANNOTATION_COMMIT_UPDATE_RPC_ID, ANNOTATION_GEOMETRY_CHUNK_SOURCE_RPC_ID, ANNOTATION_METADATA_CHUNK_SOURCE_RPC_ID, ANNOTATION_REFERENCE_ADD_RPC_ID, ANNOTATION_REFERENCE_DELETE_RPC_ID, ANNOTATION_SUBSET_GEOMETRY_CHUNK_SOURCE_RPC_ID, AnnotationGeometryChunkSpecification} from 'neuroglancer/annotation/base';
import {getAnnotationTypeRenderHandler} from 'neuroglancer/annotation/type_handler';
import {ChunkState} from 'neuroglancer/chunk_manager/base';
import {Chunk, ChunkManager, ChunkSource} from 'neuroglancer/chunk_manager/frontend';
import {getObjectKey} from 'neuroglancer/segmentation_display_state/base';
import {Position} from 'neuroglancer/navigation_state';
import {forEachVisibleSegment, getObjectKey} from 'neuroglancer/segmentation_display_state/base';
import {SliceViewSourceOptions} from 'neuroglancer/sliceview/base';
import {MultiscaleSliceViewChunkSource, SliceViewChunk, SliceViewChunkSource, SliceViewChunkSourceOptions, SliceViewSingleResolutionSource} from 'neuroglancer/sliceview/frontend';
import {StatusMessage} from 'neuroglancer/status';
import {getCenterPosition} from 'neuroglancer/ui/annotations';
import {Borrowed, Owned} from 'neuroglancer/util/disposable';
import {ENDIANNESS, Endianness} from 'neuroglancer/util/endian';
import {vec3} from 'neuroglancer/util/geom';
import * as matrix from 'neuroglancer/util/matrix';
import {NullarySignal, Signal} from 'neuroglancer/util/signal';
import {Buffer} from 'neuroglancer/webgl/buffer';
Expand Down Expand Up @@ -377,6 +382,35 @@ export function makeTemporaryChunk() {
{data: new Uint8Array(0), numPickIds: 0, typeToOffset, typeToIds, typeToIdMaps});
}

export function deserializeAnnotations(
serializedAnnotations: SerializedAnnotations, rank: number,
properties: Readonly<AnnotationPropertySpec>[]) {
const annotations: Annotation[] = [];
const annotationBuffer = serializedAnnotations.data;
let annotation: Annotation|undefined;
for (let [annotationType, annotationsOfType] of serializedAnnotations.typeToIdMaps.entries()) {
const handler = annotationTypeHandlers[annotationType as AnnotationType];
const numGeometryBytes = handler.serializedBytes(rank);
const baseOffset = annotationBuffer.byteOffset;
const dataView = new DataView(annotationBuffer.buffer);
const isLittleEndian = Endianness.LITTLE === ENDIANNESS;
const annotationPropertySerializer =
new AnnotationPropertySerializer(rank, numGeometryBytes, properties);
const annotationCount = annotationsOfType.size;
for (const [annotationId, annotationIndex] of annotationsOfType) {
annotation = handler.deserialize(
dataView,
baseOffset + annotationPropertySerializer.propertyGroupBytes[0] * annotationIndex,
isLittleEndian, rank, annotationId);
annotationPropertySerializer.deserialize(
dataView, baseOffset, annotationIndex, annotationCount, isLittleEndian,
annotation.properties = new Array(properties.length));
annotations.push(annotation);
}
}
return annotations;
}

export class MultiscaleAnnotationSource extends SharedObject implements
MultiscaleSliceViewChunkSource<AnnotationGeometryChunkSource>, AnnotationSourceSignals {
OPTIONS: {};
Expand Down Expand Up @@ -409,6 +443,120 @@ export class MultiscaleAnnotationSource extends SharedObject implements
}
}

activeAnnotations(state: AnnotationLayerState, sortByPosition: Position): [
length: number,
indexToId: (index: number) => Annotation,
idToIndex: (id: string) => number | undefined,
] {
const {segmentFilteredSources, spatiallyIndexedSources, rank, properties, relationships} = this;
const {relationshipStates} = state.displayState;
let hasVisibleSegments = false;

let currentLength = 0;
const listOffsets: number[] = []; // TODO, maybe change to index start
const serializedAnnotationsList: SerializedAnnotations[] = [];
const deserializedAnnotationsList: Annotation[][] = [];
const idToIndexMap = new Map<string, number>();

for (let i = 0; i < relationships.length; i++) {
const relationship = relationships[i];
const state = relationshipStates.get(relationship)
if (state) {
const {showMatches: {value: showMatches}, segmentationState: {value: segmentationState}} =
state;
if (!showMatches || !segmentationState) continue;
const chunks = segmentFilteredSources[i].chunks;
forEachVisibleSegment(segmentationState.segmentationGroupState.value, objectId => {
hasVisibleSegments = true;
const key = getObjectKey(objectId);
const chunk = chunks.get(key);
if (chunk !== undefined && chunk.state === ChunkState.GPU_MEMORY) {
const {data} = chunk;
if (data === undefined) return;
const {serializedAnnotations} = data;
const length = serializedAnnotations.typeToIds.reduce(
(sum, idsOfType) => sum + idsOfType.length, 0);
listOffsets.push(currentLength);
currentLength += length;
}
});
}
}
if (!hasVisibleSegments) {
for (const source of spatiallyIndexedSources) {
const {rank} = sortByPosition.coordinateSpace.value;
const sortChunk = new Uint32Array(rank);
const {chunkDataSize} = source.spec;
for (let i = 0; i < rank; i++) {
sortChunk[i] = Math.floor(sortByPosition.value[i] / chunkDataSize[i]);
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is the basic chunk calculation that doesn't match up with the loaded chunks. The idea is to start with the highest level chunks and go out till we have a chunk that is loaded. Eventually it would be nice to support neighboring chunks particularly if the global is near a chunk boundary.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For sortByPosition you are just using the globalPosition, which is not necessarily in the same coordinate space. Instead you need to transform the relevant globalPosition and localPosition to the coordinate space of the annotation layer, as done here:

neuroglancer/src/neuroglancer/ui/annotations.ts

Line 852 in c466b24

function getMousePositionInAnnotationCoordinates(

Then each scale level also has a coordinate transform.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I converted getMousePositionInAnnotationCoordinates to getPositionInAnnotationCoordinates so I could pass in any Float32Array, call that with

const point = getPositionInAnnotationCoordinates(
  globalPosition.value,
  state,
);

I then copied the logic in forEachPossibleChunk:

neuroglancer/src/neuroglancer/annotation/frontend_source.ts

Line 567 in c466b24

private forEachPossibleChunk(

To make positionToMultiscaleChunks(position: Float32Array)
https://gist.github.com/chrisj/328828d13480627961166beb16cfcd18

This uses source.multiscaleToChunkTransform which I assume is the coordinate transform that you mentioned per scale. The issue is that it only returns 0,0,0 for the largest scale. I'm not sure if I'm misunderstanding the code but what happens in the case of a single point is that totalChunks = 1; (unless on boundary), remainder = 0
chunkIndex = 0;, size = 1, remainder % size = 0, remainder = 0 / 1 so you end up with tempChunk = [0,0,0] and that chunk usually only exists for the largest scale.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you explain a bit more what the issue is? It is indeed the case that at the largest scale, there is normally just a single chunk.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I assume that MultiscaleAnnotationSource.forEachPossibleChunk is supposed run the callback across all the scales that contain the annotation.

When I follow the logic, it seems to be stuck at [0,0,0] across all scales so chunks.get(tempChunk.join()); only returns a result at the largest scale.

I am looking for the smallest scales that contain my target point.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I added this branch that shows the problem I'm experiencing
https://github.com/seung-lab/neuroglancer/tree/cj_multiscale_annotation_list_3

With this link http://127.0.0.1:8080/#!%7B%22dimensions%22:%7B%22x%22:%5B8e-9%2C%22m%22%5D%2C%22y%22:%5B8e-9%2C%22m%22%5D%2C%22z%22:%5B8e-9%2C%22m%22%5D%7D%2C%22position%22:%5B15610.5%2C22520.5%2C24095.5%5D%2C%22crossSectionScale%22:1%2C%22projectionOrientation%22:%5B-0.29930439591407776%2C-0.3578258454799652%2C0.8785958290100098%2C-0.10221008211374283%5D%2C%22projectionScale%22:12.370998330885378%2C%22layers%22:%5B%7B%22type%22:%22annotation%22%2C%22source%22:%22precomputed://gs://neuroglancer-janelia-flyem-hemibrain/v1.0/synapses%22%2C%22tab%22:%22annotations%22%2C%22shader%22:%22#uicontrol%20vec3%20preColor%20color%28default=%5C%22red%5C%22%29%5Cn#uicontrol%20vec3%20postColor%20color%28default=%5C%22blue%5C%22%29%5Cn#uicontrol%20float%20preConfidence%20slider%28min=0%2C%20max=1%2C%20default=0%29%5Cn#uicontrol%20float%20postConfidence%20slider%28min=0%2C%20max=1%2C%20default=0%29%5Cn%5Cnvoid%20main%28%29%20%7B%5Cn%20%20setColor%28defaultColor%28%29%29%3B%5Cn%20%20setEndpointMarkerColor%28%5Cn%20%20%20%20vec4%28preColor%2C%200.5%29%2C%5Cn%20%20%20%20vec4%28postColor%2C%200.5%29%29%3B%5Cn%20%20setEndpointMarkerSize%282.0%2C%202.0%29%3B%5Cn%20%20setLineWidth%282.0%29%3B%5Cn%20%20if%20%28prop_pre_synaptic_confidence%28%29%3C%20preConfidence%20%7C%7C%5Cn%20%20%20%20%20%20prop_post_synaptic_confidence%28%29%3C%20postConfidence%29%20discard%3B%5Cn%7D%5Cn%22%2C%22filterBySegmentation%22:%5B%22post_synaptic_cell%22%2C%22pre_synaptic_cell%22%5D%2C%22name%22:%22synapse%22%7D%5D%2C%22showSlices%22:false%2C%22selectedLayer%22:%7B%22size%22:426%2C%22visible%22:true%2C%22layer%22:%22synapse%22%7D%2C%22layout%22:%223d%22%2C%22statistics%22:%7B%22size%22:248%7D%7D

In the console you will see output such as

addChunk 0,0,0
addChunk 28,31,35
addChunk 14,15,17
addChunk 0,1,1
addChunk 0,0,0
addChunk 1,0,1
addChunk 29,30,36
addChunk 28,32,36
addChunk 0,0,1
... (removed 20 lines)
addChunk 2,3,4
addChunk 7,7,9
addChunk 1,1,2
addChunk 1,1,1
5 no chunk for key 0,0,0
chunk exists 0,0,0 size 17209,15302,19814

}
// currently not working well because the chunk calculation is off so we end up using a low
// level chunk
const chunk = source.chunks.get(sortChunk.join(','));
if (chunk) {
const {data} = chunk;
if (data === undefined) continue;
const {serializedAnnotations} = data;
const length =
serializedAnnotations.typeToIds.reduce((sum, idsOfType) => sum + idsOfType.length, 0);
listOffsets.push(currentLength);
currentLength += length;
const tempCenter = new Float32Array(rank);
const annotations = deserializeAnnotations(serializedAnnotations, rank, properties);
annotations.sort((a, b) => {
getCenterPosition(tempCenter, a);
const distanceA = vec3.distance(tempCenter as vec3, sortByPosition.value as vec3);
getCenterPosition(tempCenter, b);
const distanceB = vec3.distance(tempCenter as vec3, sortByPosition.value as vec3);
return distanceA - distanceB;
});
deserializedAnnotationsList[0] = annotations;
for (let i = 0; i < annotations.length; i++) {
idToIndexMap.set(annotations[0].id, i);
}
break;
}
}
}
const indexToId = (index: number) => {
if (index < currentLength) {
for (const [idx, listOffset] of listOffsets.entries()) {
const nextOffset = listOffsets[idx + 1] || Number.MAX_VALUE;
if (index < nextOffset) {
if (!deserializedAnnotationsList[idx]) {
const serializedAnnotations = serializedAnnotationsList[idx];
deserializedAnnotationsList[idx] =
deserializeAnnotations(serializedAnnotations, rank, properties);
}
return deserializedAnnotationsList[idx][index - listOffset];
}
}
}
throw new Error('OUT OF BOUNDS');
};
const idToIndex = (id: string) => {
const index = idToIndexMap.get(id);
if (index) {
return index;
}
for (const [idx, listOffset] of listOffsets.entries()) {
const serializedAnnotations = serializedAnnotationsList[idx];
if (!serializedAnnotations) continue;
const {typeToIdMaps} = serializedAnnotations;
for (const idMapForType of typeToIdMaps) {
const localIndex = idMapForType.get(id);
if (localIndex) {
return listOffset + localIndex; // TODO, is this handling SerializedAnnotations with
// multiple types correctly?
}
}
}
return undefined;
};

return [currentLength, indexToId, idToIndex];
}

hasNonSerializedProperties() {
return this.relationships.length > 0;
}
Expand Down Expand Up @@ -479,7 +627,8 @@ export class MultiscaleAnnotationSource extends SharedObject implements
}
} else {
if (newAnnotation === null) {
// Annotation has a local update already, so we need to delete it from the temporary chunk.
// Annotation has a local update already, so we need to delete it from the temporary
// chunk.
deleteAnnotation(
this.temporary.data!, annotation.type, annotation.id,
this.annotationPropertySerializers);
Expand Down
Loading
Loading