From ec07047dff5090bcd7b059aaedd149ead13c516b Mon Sep 17 00:00:00 2001 From: Rick Saccoccia Date: Tue, 26 Nov 2024 11:06:02 -0700 Subject: [PATCH 1/3] [service] sync configured events instead of checking changes of all events. Store event ids instead of names --- plugins/arcgis/service/src/ArcGISConfig.ts | 14 ++--- .../service/src/EventDeletionHandler.ts | 2 +- .../src/EventLayerProcessorOrganizer.ts | 2 +- .../arcgis/service/src/FeatureServiceAdmin.ts | 18 +++---- plugins/arcgis/service/src/LayerInfo.ts | 17 +++--- .../service/src/ObservationProcessor.ts | 48 +++++++---------- plugins/arcgis/service/src/index.ts | 52 +++++++++++++++---- 7 files changed, 90 insertions(+), 63 deletions(-) diff --git a/plugins/arcgis/service/src/ArcGISConfig.ts b/plugins/arcgis/service/src/ArcGISConfig.ts index a0904daee..dc7c7378c 100644 --- a/plugins/arcgis/service/src/ArcGISConfig.ts +++ b/plugins/arcgis/service/src/ArcGISConfig.ts @@ -1,3 +1,5 @@ +import { MageEventId } from "@ngageoint/mage.service/lib/entities/events/entities.events" + /** * Contains an arc feature service url and layers. */ @@ -8,10 +10,10 @@ export interface FeatureServiceConfig { */ url: string - /** - * Serialized ArcGISIdentityManager - */ - identityManager: string + /** + * Serialized ArcGISIdentityManager + */ + identityManager: string /** * The feature layers. @@ -35,9 +37,9 @@ export interface FeatureLayerConfig { geometryType?: string /** - * The event ids or names that sync to this arc feature layer. + * The event ids that sync to this arc feature layer. */ - events?: (number|string)[] + eventIds?: MageEventId[] } diff --git a/plugins/arcgis/service/src/EventDeletionHandler.ts b/plugins/arcgis/service/src/EventDeletionHandler.ts index 7e8181447..305485edd 100644 --- a/plugins/arcgis/service/src/EventDeletionHandler.ts +++ b/plugins/arcgis/service/src/EventDeletionHandler.ts @@ -83,7 +83,7 @@ export class EventDeletionHandler { } /** - * Called when the query is finished. It goes through the results and gathers all even Ids currently stored + * Called when the query is finished. It goes through the results and gathers all event Ids currently stored * in the arc layer. It then will remove any events from the arc layer that do not exist. * @param layerProcessor The feature layer processor. * @param result The returned results. diff --git a/plugins/arcgis/service/src/EventLayerProcessorOrganizer.ts b/plugins/arcgis/service/src/EventLayerProcessorOrganizer.ts index 05d0d024b..d05375fff 100644 --- a/plugins/arcgis/service/src/EventLayerProcessorOrganizer.ts +++ b/plugins/arcgis/service/src/EventLayerProcessorOrganizer.ts @@ -19,7 +19,7 @@ export class EventLayerProcessorOrganizer { for (const event of events) { let syncProcessors = new Array(); for (const layerProcessor of layerProcessors) { - if (layerProcessor.layerInfo.hasEvent(event.name)) { + if (layerProcessor.layerInfo.hasEvent(event.id)) { syncProcessors.push(layerProcessor); } } diff --git a/plugins/arcgis/service/src/FeatureServiceAdmin.ts b/plugins/arcgis/service/src/FeatureServiceAdmin.ts index d38fa7989..fa3fa411f 100644 --- a/plugins/arcgis/service/src/FeatureServiceAdmin.ts +++ b/plugins/arcgis/service/src/FeatureServiceAdmin.ts @@ -1,6 +1,6 @@ import { ArcGISPluginConfig } from "./ArcGISPluginConfig" import { FeatureServiceConfig, FeatureLayerConfig } from "./ArcGISConfig" -import { MageEvent, MageEventRepository } from '@ngageoint/mage.service/lib/entities/events/entities.events' +import { MageEvent, MageEventId, MageEventRepository } from '@ngageoint/mage.service/lib/entities/events/entities.events' import { Layer, Field } from "./AddLayersRequest" import { Form, FormField, FormFieldType, FormId } from '@ngageoint/mage.service/lib/entities/events/entities.events.forms' import { ObservationsTransformer } from "./ObservationsTransformer" @@ -120,21 +120,21 @@ export class FeatureServiceAdmin { } /** - * Get the layer events + * Get the Mage layer events * @param layer feature layer * @param eventRepo event repository - * @returns layer events + * @returns Mage layer events */ private async layerEvents(layer: FeatureLayerConfig, eventRepo: MageEventRepository): Promise { - const layerEvents: Set = new Set() - if (layer.events != null) { - for (const layerEvent of layer.events) { - layerEvents.add(layerEvent) + const layerEventIds: Set = new Set() + if (layer.eventIds != null) { + for (const layerEventId of layer.eventIds) { + layerEventIds.add(layerEventId) } } let mageEvents - if (layerEvents.size > 0) { + if (layerEventIds.size > 0) { mageEvents = await eventRepo.findAll() } else { mageEvents = await eventRepo.findActiveEvents() @@ -142,7 +142,7 @@ export class FeatureServiceAdmin { const events: MageEvent[] = [] for (const mageEvent of mageEvents) { - if (layerEvents.size == 0 || layerEvents.has(mageEvent.name) || layerEvents.has(mageEvent.id)) { + if (layerEventIds.size == 0 || layerEventIds.has(mageEvent.id)) { const event = await eventRepo.findById(mageEvent.id) if (event != null) { events.push(event) diff --git a/plugins/arcgis/service/src/LayerInfo.ts b/plugins/arcgis/service/src/LayerInfo.ts index 4918d7153..4d1790904 100644 --- a/plugins/arcgis/service/src/LayerInfo.ts +++ b/plugins/arcgis/service/src/LayerInfo.ts @@ -1,3 +1,4 @@ +import { MageEventId } from "@ngageoint/mage.service/lib/entities/events/entities.events"; import { LayerInfoResult, LayerField } from "./LayerInfoResult"; /** @@ -28,7 +29,7 @@ export class LayerInfo { /** * The events that are synching to this layer. */ - events: Set = new Set() + events: Set = new Set() /** * Constructor. @@ -37,12 +38,10 @@ export class LayerInfo { * @param layerInfo The layer info. * @param token The access token. */ - constructor(url: string, events: string[], layerInfo: LayerInfoResult) { + constructor(url: string, events: MageEventId[], layerInfo: LayerInfoResult) { this.url = url - if (events != undefined && events != null && events.length == 0) { - this.events.add('nothing to sync') - } - if (events != undefined || events != null) { + + if (events && events.length > 0) { for (const event of events) { this.events.add(event); } @@ -69,11 +68,11 @@ export class LayerInfo { /** * Determine if the layer is enabled for the event. - * @param event The event. + * @param eventId The event. * @return true if enabled */ - hasEvent(event: string) { - return this.events.size == 0 || this.events.has(event) + hasEvent(eventId: MageEventId) { + return this.events.size == 0 || this.events.has(eventId) } } \ No newline at end of file diff --git a/plugins/arcgis/service/src/ObservationProcessor.ts b/plugins/arcgis/service/src/ObservationProcessor.ts index c4ffed54a..f925a755e 100644 --- a/plugins/arcgis/service/src/ObservationProcessor.ts +++ b/plugins/arcgis/service/src/ObservationProcessor.ts @@ -1,4 +1,5 @@ import { PagingParameters } from '@ngageoint/mage.service/lib/entities/entities.global'; +import { MageEventId } from "@ngageoint/mage.service/lib/entities/events/entities.events"; import { MageEventRepository } from '@ngageoint/mage.service/lib/entities/events/entities.events'; import { EventScopedObservationRepository, ObservationRepositoryForEvent } from '@ngageoint/mage.service/lib/entities/observations/entities.observations'; import { UserRepository } from '@ngageoint/mage.service/lib/entities/users/entities.users'; @@ -164,8 +165,13 @@ export class ObservationProcessor { private async updateConfig(): Promise { const config = await this.safeGetConfig() - // Include form definitions while detecting changes in config - const eventForms = await this._eventRepo.findAll(); + // Include configured eventform definitions while detecting changes in config + const eventIds = config.featureServices + .flatMap(service => service.layers) + .flatMap(layer => layer.eventIds) + .filter((eventId): eventId is MageEventId => typeof eventId === 'number'); + + const eventForms = await this._eventRepo.findAllByIds(eventIds); const fullConfig = { ...config, eventForms }; const configJson = JSON.stringify(fullConfig) @@ -207,7 +213,7 @@ export class ObservationProcessor { try { const identityManager = await this._identityService.signin(service) const response = await request(service.url, { authentication: identityManager }) - this.handleFeatureService(response, service, config) + await this.handleFeatureService(response, service, config) } catch (err) { console.error(err) } @@ -235,25 +241,7 @@ export class ObservationProcessor { } for (const featureLayer of featureServiceConfig.layers) { - const eventNames: string[] = [] - const events = featureLayer.events - if (events != null) { - for (const event of events) { - const eventId = Number(event); - if (isNaN(eventId)) { - eventNames.push(String(event)); - } else { - const mageEvent = await this._eventRepo.findById(eventId) - if (mageEvent != null) { - eventNames.push(mageEvent.name); - } - } - } - } - if (eventNames.length > 0) { - featureLayer.events = eventNames - } - + // TODO - this used to convert event ids to names and set back on featureLayer.events. What is impact of not doing? const layer = serviceLayers.get(featureLayer.layer) let layerId = undefined @@ -270,7 +258,7 @@ export class ObservationProcessor { const featureService = new FeatureService(console, featureServiceConfig, identityManager) const layerInfo = await featureService.queryLayerInfo(layerId); const url = `${featureServiceConfig.url}/${layerId}`; - this.handleLayerInfo(url, featureServiceConfig, featureLayer, layerInfo, config); + await this.handleLayerInfo(url, featureServiceConfig, featureLayer, layerInfo, config); } } } @@ -286,10 +274,10 @@ export class ObservationProcessor { */ private async handleLayerInfo(url: string, featureServiceConfig: FeatureServiceConfig, featureLayer: FeatureLayerConfig, layerInfo: LayerInfoResult, config: ArcGISPluginConfig) { if (layerInfo.geometryType != null) { - const events = featureLayer.events as string[] const admin = new FeatureServiceAdmin(config, this._identityService, this._console) + const eventIds = featureLayer.eventIds || [] await admin.updateLayer(featureServiceConfig, featureLayer, layerInfo, this._eventRepo) - const info = new LayerInfo(url, events, layerInfo) + const info = new LayerInfo(url, eventIds, layerInfo) const identityManager = await this._identityService.signin(featureServiceConfig) const layerProcessor = new FeatureLayerProcessor(info, config, identityManager, this._console); this._layerProcessors.push(layerProcessor); @@ -310,9 +298,13 @@ export class ObservationProcessor { layerProcessor.processPendingUpdates(); } this._console.info('ArcGIS plugin processing new observations...'); - const activeEvents = await this._eventRepo.findActiveEvents(); - this._eventDeletionHandler.checkForEventDeletion(activeEvents, this._layerProcessors, this._firstRun); - const eventsToProcessors = this._organizer.organize(activeEvents, this._layerProcessors); + const enabledEvents = (await this._eventRepo.findActiveEvents()).filter(event => + this._layerProcessors.some(layerProcessor => + layerProcessor.layerInfo.hasEvent(event.id) + ) + ); + this._eventDeletionHandler.checkForEventDeletion(enabledEvents, this._layerProcessors, this._firstRun); + const eventsToProcessors = this._organizer.organize(enabledEvents, this._layerProcessors); const nextQueryTime = Date.now(); for (const pair of eventsToProcessors) { this._console.info('ArcGIS getting newest observations for event ' + pair.event.name); diff --git a/plugins/arcgis/service/src/index.ts b/plugins/arcgis/service/src/index.ts index b70e32333..97e32caa8 100644 --- a/plugins/arcgis/service/src/index.ts +++ b/plugins/arcgis/service/src/index.ts @@ -4,9 +4,10 @@ import { ObservationRepositoryToken } from '@ngageoint/mage.service/lib/plugins. import { MageEventRepositoryToken } from '@ngageoint/mage.service/lib/plugins.api/plugins.api.events' import { UserRepositoryToken } from '@ngageoint/mage.service/lib/plugins.api/plugins.api.users' import { SettingPermission } from '@ngageoint/mage.service/lib/entities/authorization/entities.permissions' +import { MageEventId } from '@ngageoint/mage.service/lib/entities/events/entities.events' import { ObservationProcessor } from './ObservationProcessor' import { ArcGISIdentityManager, request } from "@esri/arcgis-rest-request" -import { FeatureServiceConfig } from './ArcGISConfig' +import { FeatureServiceConfig, FeatureLayerConfig } from './ArcGISConfig' import { URL } from "node:url" import express from 'express' import { ArcGISIdentityService, createArcGISIdentityService, getPortalUrl } from './ArcGISService' @@ -166,16 +167,49 @@ const arcgisPluginHooks: InitPluginHook = { const config = await stateRepo.get() const { featureServices: updatedServices, ...updateConfig } = req.body - // Map exisiting identityManager, client does not send this - const featureServices: FeatureServiceConfig[] = updatedServices.map((updateService: FeatureServiceConfig) => { - const existingService = config.featureServices.find((featureService: FeatureServiceConfig) => featureService.url === updateService.url) + + // Convert event names to event IDs + // Fetch all events and create a mapping of event names to event IDs + const allEvents = await eventRepo.findAll(); + const eventNameToIdMap = new Map(); + allEvents.forEach(event => { + eventNameToIdMap.set(event.name, event.id); + }); + + // Process the incoming feature services with eventIds instead of event names + const featureServices: FeatureServiceConfig[] = updatedServices.map((updateService: any) => { + const existingService = config.featureServices.find( + (featureService: FeatureServiceConfig) => featureService.url === updateService.url + ); + + // Process layers + const layers: FeatureLayerConfig[] = updateService.layers.map((layer: any) => { + // Extract event names from the incoming layer data + const eventNames: string[] = layer.events || []; + + // Convert event names to event IDs using the mapping + const eventIds = eventNames + .map(eventName => eventNameToIdMap.get(eventName)) + .filter((id): id is MageEventId => id !== undefined); + + // Construct the FeatureLayerConfig with eventIds + const featureLayerConfig: FeatureLayerConfig = { + layer: layer.layer, + geometryType: layer.geometryType, + eventIds: eventIds, + }; + + return featureLayerConfig; + }); + return { url: updateService.url, - layers: updateService.layers, - identityManager: existingService?.identityManager - } - }) - + layers: layers, + // Map exisiting identityManager, client does not send this + identityManager: existingService?.identityManager, + }; + }); + await stateRepo.patch({ ...updateConfig, featureServices }) // Sync configuration with feature servers by restarting observation processor From fdaa660a2c787361f2cd16c10c7f2a5884a756fd Mon Sep 17 00:00:00 2001 From: Rick Saccoccia Date: Wed, 27 Nov 2024 14:30:54 -0700 Subject: [PATCH 2/3] [server] Fix query syntax error --- plugins/arcgis/service/src/FeatureQuerier.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/arcgis/service/src/FeatureQuerier.ts b/plugins/arcgis/service/src/FeatureQuerier.ts index 579fe48c8..a1c98cf74 100644 --- a/plugins/arcgis/service/src/FeatureQuerier.ts +++ b/plugins/arcgis/service/src/FeatureQuerier.ts @@ -56,7 +56,7 @@ export class FeatureQuerier { if (this._config.eventIdField == null) { queryUrl.searchParams.set('where', `${this._config.observationIdField} LIKE '${observationId}${this._config.idSeparator}%'`); } else { - queryUrl.searchParams.set('where', `${this._config.observationIdField} = ${observationId}`); + queryUrl.searchParams.set('where', `${this._config.observationIdField} = '${observationId}'`); } queryUrl.searchParams.set('outFields', this.outFields(fields)) queryUrl.searchParams.set('returnGeometry', geometry === false ? 'false' : 'true') From e98eb8228508a96ded20fbaf55c0df2f99429aee Mon Sep 17 00:00:00 2001 From: Rick Saccoccia Date: Sat, 30 Nov 2024 15:01:11 -0700 Subject: [PATCH 3/3] [service] fix event deletion and firstRun logic --- plugins/arcgis/service/src/EventDeletionHandler.ts | 4 ++++ plugins/arcgis/service/src/ObservationProcessor.ts | 4 +--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/plugins/arcgis/service/src/EventDeletionHandler.ts b/plugins/arcgis/service/src/EventDeletionHandler.ts index 305485edd..691c69072 100644 --- a/plugins/arcgis/service/src/EventDeletionHandler.ts +++ b/plugins/arcgis/service/src/EventDeletionHandler.ts @@ -34,6 +34,10 @@ export class EventDeletionHandler { this._config = config; } + public updateConfig(newConfig: ArcGISPluginConfig): void { + this._config = newConfig; + } + /** * * @param activeEvents The current set of active events. diff --git a/plugins/arcgis/service/src/ObservationProcessor.ts b/plugins/arcgis/service/src/ObservationProcessor.ts index f925a755e..5abbc4ca4 100644 --- a/plugins/arcgis/service/src/ObservationProcessor.ts +++ b/plugins/arcgis/service/src/ObservationProcessor.ts @@ -178,11 +178,10 @@ export class ObservationProcessor { if (this._previousConfig == null || this._previousConfig != configJson) { this._transformer = new ObservationsTransformer(config, console); this._geometryChangeHandler = new GeometryChangedHandler(this._transformer); - this._eventDeletionHandler = new EventDeletionHandler(this._console, config); + this._eventDeletionHandler.updateConfig(config); this._layerProcessors = []; await this.getFeatureServiceLayers(config); this._previousConfig = configJson - this._firstRun = true; } return config } @@ -192,7 +191,6 @@ export class ObservationProcessor { */ async start() { this._isRunning = true; - this._firstRun = true; this.processAndScheduleNext(); }