diff --git a/README.md b/README.md index eea8e3907..398cb4e65 100644 --- a/README.md +++ b/README.md @@ -295,8 +295,15 @@ npm run build After building the core packages, install them as dependencies in the `instance` package. ```bash cd instance +npm i +``` + +In the case that the dev dependencies need to be over ridden (eg nga-msi plugin) +```bash +cd instance npm i --omit dev ../service ../web-app/dist ../plugins/nga-msi ``` + The project's root [`package.json`](./package.json) provides some convenience script entries to install, build, and run the MAGE server components, however, those are deprecated and will likely go away after migrating to NPM 7+'s [workspaces](https://docs.npmjs.com/cli/v8/using-npm/workspaces) feature. diff --git a/plugins/arcgis/README.md b/plugins/arcgis/README.md new file mode 100644 index 000000000..6a0f1a826 --- /dev/null +++ b/plugins/arcgis/README.md @@ -0,0 +1,19 @@ +# Build with arcgis plugin + +Follow the instructions in the root README. After completing the web-app package install and build in the 'Building from source' section: + +Build arcgis plugin: +```bash +cd plugins/arcgis/service +npm ci +npm link ../../../service # **IMPORTANT** see root README +npm run build +``` +```bash +cd plugins/arcgis/web-app +npm ci +npm link ../../../web-app # **IMPORTANT** see root README +npm run build +``` + +Continue to install dependencies in the `instance` package as instructed in the root README. \ No newline at end of file diff --git a/plugins/arcgis/service/package-lock.json b/plugins/arcgis/service/package-lock.json index 246d648d1..4b696144d 100644 --- a/plugins/arcgis/service/package-lock.json +++ b/plugins/arcgis/service/package-lock.json @@ -12,7 +12,7 @@ "@esri/arcgis-rest-feature-service": "^4.0.6", "@esri/arcgis-rest-request": "^4.2.3", "@terraformer/arcgis": "2.1.2", - "form-data": "^4.0.0" + "form-data": "^4.0.1" }, "devDependencies": { "@types/express": "^4.17.21", @@ -16992,9 +16992,10 @@ } }, "node_modules/form-data": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", - "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.1.tgz", + "integrity": "sha512-tzN8e4TX8+kkxGPK8D5u0FNmjPUjw3lwC9lSLxxoB/+GtsJG91CO8bSWy73APlgAZzZbXEYZJuxjkHH2w+Ezhw==", + "license": "MIT", "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", diff --git a/plugins/arcgis/service/package.json b/plugins/arcgis/service/package.json index c1653745a..6d18e5827 100644 --- a/plugins/arcgis/service/package.json +++ b/plugins/arcgis/service/package.json @@ -41,7 +41,7 @@ "@esri/arcgis-rest-feature-service": "^4.0.6", "@esri/arcgis-rest-request": "^4.2.3", "@terraformer/arcgis": "2.1.2", - "form-data": "^4.0.0" + "form-data": "^4.0.1" }, "peerDependencies": { "@ngageoint/mage.service": "^6.2.9 || ^6.3.0-beta", diff --git a/plugins/arcgis/service/src/ArcGISConfig.ts b/plugins/arcgis/service/src/ArcGISConfig.ts index 5d3bb6977..a0904daee 100644 --- a/plugins/arcgis/service/src/ArcGISConfig.ts +++ b/plugins/arcgis/service/src/ArcGISConfig.ts @@ -8,25 +8,10 @@ export interface FeatureServiceConfig { */ url: string - /** - * Username and password for ArcGIS authentication - */ - auth?: ArcGISAuthConfig - - /** - * Create layers that don't exist - */ - createLayers?: boolean - - /** - * The administration url to the arc feature service. - */ - adminUrl?: string - - /** - * Administration access token - */ - adminToken?: string + /** + * Serialized ArcGISIdentityManager + */ + identityManager: string /** * The feature layers. @@ -49,104 +34,12 @@ export interface FeatureLayerConfig { */ geometryType?: string - /** - * Access token - */ - token?: string // TODO - can this be removed? Will Layers have a token too? /** * The event ids or names that sync to this arc feature layer. */ events?: (number|string)[] - - /** - * Add layer fields from form fields - */ - addFields?: boolean - - /** - * Delete editable layer fields missing from form fields - */ - deleteFields?: boolean - } -export enum AuthType { - Token = 'token', - UsernamePassword = 'usernamePassword', - OAuth = 'oauth' - } - - -/** - * Contains token-based authentication configuration. - */ -export interface TokenAuthConfig { - type: AuthType.Token - token: string - authTokenExpires?: string -} - -/** - * Contains username and password for ArcGIS server authentication. - */ -export interface UsernamePasswordAuthConfig { - type: AuthType.UsernamePassword - /** - * The username for authentication. - */ - username: string - - /** - * The password for authentication. - */ - password: string -} - -/** - * Contains OAuth authentication configuration. - */ -export interface OAuthAuthConfig { - - type: AuthType.OAuth - - /** - * The Client Id for OAuth - */ - clientId: string - - /** - * The redirectUri for OAuth - */ - redirectUri?: string - - /** - * The temporary auth token for OAuth - */ - authToken?: string - - /** - * The expiration date for the temporary token - */ - authTokenExpires?: number - - /** - * The Refresh token for OAuth - */ - refreshToken?: string - - /** - * The expiration date for the Refresh token - */ - refreshTokenExpires?: number -} - -/** - * Union type for authentication configurations. - */ -export type ArcGISAuthConfig = - | TokenAuthConfig - | UsernamePasswordAuthConfig - | OAuthAuthConfig /** * Attribute configurations diff --git a/plugins/arcgis/service/src/ArcGISIdentityManagerFactory.ts b/plugins/arcgis/service/src/ArcGISIdentityManagerFactory.ts deleted file mode 100644 index dafadec96..000000000 --- a/plugins/arcgis/service/src/ArcGISIdentityManagerFactory.ts +++ /dev/null @@ -1,118 +0,0 @@ -import { ArcGISIdentityManager, request } from "@esri/arcgis-rest-request" -import { ArcGISAuthConfig, AuthType, FeatureServiceConfig, OAuthAuthConfig, TokenAuthConfig, UsernamePasswordAuthConfig } from './ArcGISConfig' -import { ObservationProcessor } from "./ObservationProcessor"; - -interface ArcGISIdentityManagerFactory { - create(portal: string, server: string, config: ArcGISAuthConfig, processor?: ObservationProcessor): Promise -} - -const OAuthIdentityManagerFactory: ArcGISIdentityManagerFactory = { - async create(portal: string, server: string, auth: OAuthAuthConfig, processor: ObservationProcessor): Promise { - console.debug('Client ID provided for authentication') - const { clientId, authToken, authTokenExpires, refreshToken, refreshTokenExpires } = auth - - if (authToken && new Date(authTokenExpires || 0) > new Date()) { - return ArcGISIdentityManager.fromToken({ - clientId: clientId, - token: authToken, - tokenExpires: new Date(authTokenExpires || 0), - portal: portal, - server: server - }) - } else if (refreshToken && new Date(refreshTokenExpires || 0) > new Date()) { - // TODO: find a way without using constructor nor httpClient - const url = `${portal}/oauth2/token?client_id=${clientId}&refresh_token=${refreshToken}&grant_type=refresh_token` - try { - const response = await request(url, { - httpMethod: 'GET' - }); - - // Update authToken to new token - const config = await processor.safeGetConfig(); - let service = config.featureServices.find(service => service.url === portal)?.auth as OAuthAuthConfig; - const date = new Date(); - date.setSeconds(date.getSeconds() + response.expires_in || 0); - service = { - ...service, - authToken: response.access_token, - authTokenExpires: date.getTime() - } - - await processor.putConfig(config) - return ArcGISIdentityManager.fromToken({ - clientId: clientId, - token: response.access_token, - tokenExpires: date, - portal: portal - }); - } catch (error) { - throw new Error('Error occurred when using refresh token') - } - - } else { - // TODO the config, we need to let the user know UI side they need to authenticate again - throw new Error('Refresh token missing or expired') - } - } -} - -const TokenIdentityManagerFactory: ArcGISIdentityManagerFactory = { - async create(portal: string, server: string, auth: TokenAuthConfig): Promise { - console.debug('Token provided for authentication') - const identityManager = await ArcGISIdentityManager.fromToken({ - token: auth.token, - portal: portal, - server: server, - // TODO: what do we really want to do here? esri package seems to need this optional parameter. - // Use authTokenExpires if defined, otherwise set to now plus a day - tokenExpires: auth.authTokenExpires ? new Date(auth.authTokenExpires) : new Date(Date.now() + 24 * 60 * 60 * 1000) - }) - return identityManager - } -} - -const UsernamePasswordIdentityManagerFactory: ArcGISIdentityManagerFactory = { - async create(portal: string, server: string, auth: UsernamePasswordAuthConfig): Promise { - console.debug('console and password provided for authentication, username:' + auth?.username) - const identityManager = await ArcGISIdentityManager.signIn({ username: auth?.username, password: auth?.password, portal }) - return identityManager - } -} - -const authConfigMap: { [key: string]: ArcGISIdentityManagerFactory } = { - [AuthType.OAuth]: OAuthIdentityManagerFactory, - [AuthType.Token]: TokenIdentityManagerFactory, - [AuthType.UsernamePassword]: UsernamePasswordIdentityManagerFactory -} - -export function getIdentityManager( - config: FeatureServiceConfig, - processor: ObservationProcessor -): Promise { - const auth = config.auth - const authType = config.auth?.type - if (!auth || !authType) { - throw new Error('Auth type is undefined') - } - const factory = authConfigMap[authType] - if (!factory) { - throw new Error(`No factory found for type ${authType}`) - } - return factory.create(getPortalUrl(config.url), getServerUrl(config.url), auth, processor) -} - - -export function getPortalUrl(featureService: FeatureServiceConfig | string): string { - const url = getFeatureServiceUrl(featureService) - return `https://${url.hostname}/arcgis/sharing/rest` -} - -export function getServerUrl(featureService: FeatureServiceConfig | string): string { - const url = getFeatureServiceUrl(featureService) - return `https://${url.hostname}/arcgis` -} - -export function getFeatureServiceUrl(featureService: FeatureServiceConfig | string): URL { - const url = typeof featureService === 'string' ? featureService : featureService.url - return new URL(url) -} \ No newline at end of file diff --git a/plugins/arcgis/service/src/ArcGISPluginConfig.ts b/plugins/arcgis/service/src/ArcGISPluginConfig.ts index 831ce8b9a..78f8a8a26 100644 --- a/plugins/arcgis/service/src/ArcGISPluginConfig.ts +++ b/plugins/arcgis/service/src/ArcGISPluginConfig.ts @@ -145,7 +145,7 @@ export const defaultArcGISPluginConfig = Object.freeze({ textAreaFieldLength: 256, observationIdField: 'description', idSeparator: '-', - // eventIdField: 'event_id', + eventIdField: 'event_id', lastEditedDateField: 'last_edited_date', eventNameField: 'event_name', userIdField: 'user_id', diff --git a/plugins/arcgis/service/src/ArcGISService.ts b/plugins/arcgis/service/src/ArcGISService.ts new file mode 100644 index 000000000..7b53824f5 --- /dev/null +++ b/plugins/arcgis/service/src/ArcGISService.ts @@ -0,0 +1,57 @@ +import { ArcGISIdentityManager } from '@esri/arcgis-rest-request' +import { FeatureServiceConfig } from './ArcGISConfig' +import { PluginStateRepository } from '@ngageoint/mage.service/lib/plugins.api' + +export interface ArcGISIdentityService { + signin(featureService: FeatureServiceConfig): Promise + updateIndentityManagers(): Promise +} + +export function createArcGISIdentityService( + stateRepo: PluginStateRepository +): ArcGISIdentityService { + const identityManagerCache: Map> = new Map() + + return { + async signin(featureService: FeatureServiceConfig): Promise { + let cached = await identityManagerCache.get(featureService.url) + if (!cached) { + const identityManager = ArcGISIdentityManager.deserialize(featureService.identityManager) + const promise = identityManager.getUser().then(() => identityManager) + identityManagerCache.set(featureService.url, promise) + return promise + } else { + return cached + } + }, + async updateIndentityManagers() { + const config = await stateRepo.get() + for (let [url, persistedIdentityManagerPromise] of identityManagerCache) { + const persistedIdentityManager = await persistedIdentityManagerPromise + const featureService: FeatureServiceConfig | undefined = config.featureServices.find((service: FeatureServiceConfig) => service.url === url) + if (featureService) { + const identityManager = ArcGISIdentityManager.deserialize(featureService.identityManager) + if (identityManager.token !== persistedIdentityManager.token || identityManager.refreshToken !== persistedIdentityManager.refreshToken) { + featureService.identityManager = persistedIdentityManager.serialize() + await stateRepo.put(config) + } + } + } + } + } +} + +export function getPortalUrl(featureService: FeatureServiceConfig | string): string { + const url = getFeatureServiceUrl(featureService) + return `https://${url.hostname}/arcgis/sharing/rest` +} + +export function getServerUrl(featureService: FeatureServiceConfig | string): string { + const url = getFeatureServiceUrl(featureService) + return `https://${url.hostname}/arcgis` +} + +export function getFeatureServiceUrl(featureService: FeatureServiceConfig | string): URL { + const url = typeof featureService === 'string' ? featureService : featureService.url + return new URL(url) +} \ No newline at end of file diff --git a/plugins/arcgis/service/src/FeatureLayerProcessor.ts b/plugins/arcgis/service/src/FeatureLayerProcessor.ts index c5c82b478..60028c2da 100644 --- a/plugins/arcgis/service/src/FeatureLayerProcessor.ts +++ b/plugins/arcgis/service/src/FeatureLayerProcessor.ts @@ -5,7 +5,7 @@ import { LayerInfo } from "./LayerInfo"; import { ObservationBinner } from "./ObservationBinner"; import { ObservationBins } from "./ObservationBins"; import { ObservationsSender } from "./ObservationsSender"; - +import { ArcGISIdentityManager } from "@esri/arcgis-rest-request"; /** * Processes new, updated, and deleted observations and sends the changes to a specific arc feature layer. */ @@ -42,12 +42,12 @@ export class FeatureLayerProcessor { * @param config Contains certain parameters that can be configured. * @param console Used to log messages to the console. */ - constructor(layerInfo: LayerInfo, config: ArcGISPluginConfig, console: Console) { + constructor(layerInfo: LayerInfo, config: ArcGISPluginConfig, identityManager: ArcGISIdentityManager, console: Console) { this.layerInfo = layerInfo; this.lastTimeStamp = 0; - this.featureQuerier = new FeatureQuerier(layerInfo, config, console); + this.featureQuerier = new FeatureQuerier(layerInfo, config, identityManager,console); this._binner = new ObservationBinner(layerInfo, this.featureQuerier, config); - this.sender = new ObservationsSender(layerInfo, config, console); + this.sender = new ObservationsSender(layerInfo, config, identityManager, console); } /** @@ -85,7 +85,7 @@ export class FeatureLayerProcessor { for (const arcObservation of observations.deletions) { if (this.layerInfo.geometryType == arcObservation.esriGeometryType) { - this.sender.sendDelete(arcObservation.id) + this.sender.sendDelete(Number(arcObservation.id)); } } } diff --git a/plugins/arcgis/service/src/FeatureQuerier.ts b/plugins/arcgis/service/src/FeatureQuerier.ts index b32402c5c..579fe48c8 100644 --- a/plugins/arcgis/service/src/FeatureQuerier.ts +++ b/plugins/arcgis/service/src/FeatureQuerier.ts @@ -1,22 +1,17 @@ import { ArcGISPluginConfig } from "./ArcGISPluginConfig"; -import { HttpClient } from "./HttpClient"; import { LayerInfo } from "./LayerInfo"; import { QueryObjectResult } from "./QueryObjectResult"; +import { ArcGISIdentityManager, request } from "@esri/arcgis-rest-request"; /** * Performs various queries on observations for a specific arc feature layer. */ export class FeatureQuerier { - /** - * Used to query the arc server to figure out if an observation exists. - */ - private _httpClient: HttpClient; - /** * The query url to find out if an observations exists on the server. */ - private _url: string; + private _url: URL; /** * Used to log to console. @@ -28,15 +23,23 @@ export class FeatureQuerier { */ private _config: ArcGISPluginConfig; + /** + * An instance of `ArcGISIdentityManager` used to manage authentication and identity for ArcGIS services. + * This private member handles the authentication process, ensuring that requests to ArcGIS services + * are properly authenticated using the credentials provided. + */ + private _identityManager: ArcGISIdentityManager; + /** * Constructor. * @param layerInfo The layer info. * @param config The plugins configuration. * @param console Used to log to the console. */ - constructor(layerInfo: LayerInfo, config: ArcGISPluginConfig, console: Console) { - this._httpClient = new HttpClient(console, layerInfo.token); - this._url = layerInfo.url + '/query?where='; + constructor(layerInfo: LayerInfo, config: ArcGISPluginConfig, identityManager: ArcGISIdentityManager, console: Console) { + this._identityManager = identityManager; + this._url = new URL(layerInfo.url); + this._url.pathname += '/query'; this._console = console; this._config = config; } @@ -48,19 +51,23 @@ export class FeatureQuerier { * @param fields fields to query, all fields if not provided * @param geometry query the geometry, default is true */ - queryObservation(observationId: string, response: (result: QueryObjectResult) => void, fields?: string[], geometry?: boolean) { - let queryUrl = this._url + this._config.observationIdField + async queryObservation(observationId: string, response: (result: QueryObjectResult) => void, fields?: string[], geometry?: boolean) { + const queryUrl = new URL(this._url) if (this._config.eventIdField == null) { - queryUrl += ' LIKE \'' + observationId + this._config.idSeparator + '%\'' + queryUrl.searchParams.set('where', `${this._config.observationIdField} LIKE '${observationId}${this._config.idSeparator}%'`); } else { - queryUrl += '=\'' + observationId + '\'' + queryUrl.searchParams.set('where', `${this._config.observationIdField} = ${observationId}`); } - queryUrl += this.outFields(fields) + this.returnGeometry(geometry) - this._httpClient.sendGetHandleResponse(queryUrl, (chunk) => { - this._console.info('ArcGIS response for ' + queryUrl + ' ' + chunk) - const result = JSON.parse(chunk) as QueryObjectResult - response(result) + queryUrl.searchParams.set('outFields', this.outFields(fields)) + queryUrl.searchParams.set('returnGeometry', geometry === false ? 'false' : 'true') + this._console.info('ArcGIS query: ' + queryUrl) + + const queryResponse = await request(queryUrl.toString(), { + authentication: this._identityManager, + params: { f: 'json' } }); + + response(queryResponse as QueryObjectResult); } /** @@ -69,13 +76,20 @@ export class FeatureQuerier { * @param fields fields to query, all fields if not provided * @param geometry query the geometry, default is true */ - queryObservations(response: (result: QueryObjectResult) => void, fields?: string[], geometry?: boolean) { - let queryUrl = this._url + this._config.observationIdField + ' IS NOT NULL' + this.outFields(fields) + this.returnGeometry(geometry) - this._httpClient.sendGetHandleResponse(queryUrl, (chunk) => { - this._console.info('ArcGIS response for ' + queryUrl + ' ' + chunk) - const result = JSON.parse(chunk) as QueryObjectResult - response(result) + async queryObservations(response: (result: QueryObjectResult) => void, fields?: string[], geometry?: boolean) { + const queryUrl = new URL(this._url) + queryUrl.searchParams.set('where', `${this._config.observationIdField} IS NOT NULL`); + queryUrl.searchParams.set('outFields', this.outFields(fields)); + queryUrl.searchParams.set('returnGeometry', geometry === false ? 'false' : 'true'); + + this._console.info('ArcGIS query: ' + queryUrl) + + const queryResponse = await request(queryUrl.toString(), { + authentication: this._identityManager, + params: { f: 'json' } }); + + response(queryResponse as QueryObjectResult); } /** @@ -83,13 +97,25 @@ export class FeatureQuerier { * @param response Function called once query is complete. * @param field field to query */ - queryDistinct(response: (result: QueryObjectResult) => void, field: string) { - let queryUrl = this._url + field + ' IS NOT NULL&returnDistinctValues=true' + this.outFields([field]) + this.returnGeometry(false) - this._httpClient.sendGetHandleResponse(queryUrl, (chunk) => { - this._console.info('ArcGIS response for ' + queryUrl + ' ' + chunk) - const result = JSON.parse(chunk) as QueryObjectResult - response(result) - }); + async queryDistinct(response: (result: QueryObjectResult) => void, field: string) { + const queryUrl = new URL(this._url); + queryUrl.searchParams.set('where', `${field} IS NOT NULL`); + queryUrl.searchParams.set('returnDistinctValues', 'true'); + queryUrl.searchParams.set('outFields', this.outFields([field])); + queryUrl.searchParams.set('returnGeometry', 'false'); + this._console.info('ArcGIS query: ' + queryUrl) + + try { + const queryResponse = await request(queryUrl.toString(), { + authentication: this._identityManager, + params: { f: 'json' } + + }); + + response(queryResponse as QueryObjectResult); + } catch (err) { + console.error("could not query", err) + } } /** @@ -98,31 +124,11 @@ export class FeatureQuerier { * @returns out fields */ private outFields(fields?: string[]): string { - let outFields = '&outFields=' if (fields != null && fields.length > 0) { - for (let i = 0; i < fields.length; i++) { - if (i > 0) { - outFields += "," - } - outFields += fields[i] - } - } else{ - outFields += '*' - } - return outFields - } - - /** - * Build the return geometry query parameter - * @param fields query fields - * @returns out fields - */ - private returnGeometry(geometry?: boolean): string { - let returnGeometry = '' - if (geometry != null && !geometry) { - returnGeometry = '&returnGeometry=false' + return fields.join(','); + } else { + return '*'; } - return returnGeometry } } \ No newline at end of file diff --git a/plugins/arcgis/service/src/FeatureService.ts b/plugins/arcgis/service/src/FeatureService.ts index 51237c9fd..aabf76738 100644 --- a/plugins/arcgis/service/src/FeatureService.ts +++ b/plugins/arcgis/service/src/FeatureService.ts @@ -1,81 +1,202 @@ -import { LayerInfoResult } from "./LayerInfoResult"; -import { FeatureServiceResult } from "./FeatureServiceResult"; -import { HttpClient } from "./HttpClient"; +import { ArcGISIdentityManager, request } from "@esri/arcgis-rest-request" +import { queryFeatures, applyEdits, IQueryFeaturesOptions } from "@esri/arcgis-rest-feature-service"; +import { FeatureServiceConfig } from "./ArcGISConfig"; /** * Queries arc feature services and layers. */ export class FeatureService { - /** - * Used to make the get request about the feature layer. - */ - private _httpClient: HttpClient; - - /** - * Used to log messages. - */ private _console: Console; + private _config: FeatureServiceConfig; + private _identityManager: ArcGISIdentityManager; - /** - * Constructor. - * @param console Used to log messages. - * @param token The access token. - */ - constructor(console: Console, token?: string) { - this._httpClient = new HttpClient(console, token); + constructor(console: Console, config: FeatureServiceConfig, identityManager: ArcGISIdentityManager) { + this._config = config; + this._identityManager = identityManager; this._console = console; } - /** - * Queries an arc feature service. - * @param url The url to the arc feature layer. - * @param callback Function to call once response has been received and parsed. - */ - queryFeatureService(url: string, callback: (featureService: FeatureServiceResult) => void) { - this._httpClient.sendGetHandleResponse(url, this.parseFeatureService(url, callback)) - } + // TODO this entire class is a Work in Progress and not used. Currently using @esri/arcgis-rest-request and not arcgis-rest-js. + // By finishing this class, we can transition from low-level generic requests that leverage ArcGISIdentityManager for auth to higher-level strongly typed requests. - /** - * Parses the response from the feature service request and sends to the callback. - * @param url The url to the arc feature layer. - * @param callback The callback to call and send the feature service to. - */ - private parseFeatureService(url: string, callback: (featureService: FeatureServiceResult) => void) { - return (chunk: any) => { - this._console.log('Feature Service. url: ' + url + ', response: ' + chunk) - try { - const service = JSON.parse(chunk) as FeatureServiceResult - callback(service) - } catch (e) { - this._console.error(e) - } - } - } - /** - * Queries an arc feature layer to get info on the layer. - * @param url The url to the arc feature layer. - * @param infoCallback Function to call once response has been received and parsed. - */ - queryLayerInfo(url: string, infoCallback: (layerInfo: LayerInfoResult) => void) { - this._httpClient.sendGetHandleResponse(url, this.parseLayerInfo(url, infoCallback)); - } + // Query features using arcgis-rest-js's queryFeatures + async queryFeatureService(whereClause: string): Promise { + const queryParams = { + url: this._config.url, + where: whereClause, + authentication: this._identityManager + } as IQueryFeaturesOptions; + + this._console.log('Querying feature service with params:', queryParams); + + try { + const response = await queryFeatures(queryParams); + return response; + } catch (error) { + this._console.error('Error details:', error); + throw new Error(`Error querying feature service: ${(error as any).message}`); + } + // try { + // const response = await queryFeatures({ + // url: this._config.url, + // where: whereClause, + // authentication: this._identityManager, + // // outFields: '*', + // f: 'json', + // }); + // return response; + // } catch (error) { + // throw new Error(`Error querying feature service: ${error}`); + // } + } - /** - * Parses the response from the request and sends the layer info to the callback. - * @param url The url to the feature layer. - * @param infoCallback The callback to call and send the layer info to. - */ - private parseLayerInfo(url: string, infoCallback: (layerInfo: LayerInfoResult) => void) { - return (chunk: any) => { - this._console.log('Query Layer. url: ' + url + ', response: ' + chunk) - try { - const layerInfo = JSON.parse(chunk) as LayerInfoResult - infoCallback(layerInfo) - } catch (e) { - this._console.error(e) - } - } - } + // Generic method to query layer info + async queryLayerInfo(layerId: string | number): Promise { + const url = `${this._config.url}/${layerId}`; + try { + const response = await request(url, { + authentication: this._identityManager, + params: { f: 'json' }, + }); + return response; + } catch (error) { + throw new Error(`Error querying layer info: ${error}`); + } + } + + // Add feature using applyEdits + async addFeature(feature: any): Promise { + try { + const response = await applyEdits({ + url: this._config.url, + adds: [feature], + authentication: this._identityManager, + }); + return response; + } catch (error) { + throw new Error(`Error adding feature: ${error}`); + } + } + + // Update feature using applyEdits + async updateFeature(feature: any): Promise { + try { + const response = await applyEdits({ + url: this._config.url, + updates: [feature], + authentication: this._identityManager, + }); + return response; + } catch (error) { + throw new Error(`Error updating feature: ${error}`); + } + } + + // Delete feature using applyEdits + async deleteFeature(objectId: string | number): Promise { + try { + const response = await applyEdits({ + url: this._config.url, + deletes: [typeof objectId === 'number' ? objectId : parseInt(objectId as string, 10)], + authentication: this._identityManager, + }); + return response; + } catch (error) { + throw new Error(`Error deleting feature: ${error}`); + } + } + + // Batch operation using applyEdits + async applyEditsBatch(edits: { add?: any[], update?: any[], delete?: any[] }): Promise { + try { + const response = await applyEdits({ + url: this._config.url, + adds: edits.add || [], + updates: edits.update || [], + deletes: edits.delete || [], + authentication: this._identityManager, + }); + return response; + } catch (error) { + throw new Error(`Error applying edits: ${error}`); + } + } + + // /** + // * Queries an arc feature service. + // * @param url The url to the arc feature layer. + // * @param callback Function to call once response has been received and parsed. + // */ + // async queryFeatureService(config: FeatureServiceConfig, callback: (featureService: FeatureServiceResult) => void) { + // const httpClient = new HttpClient(this._console) + // try { + // const identityManager = await getIdentityManager(config, httpClient) + // const response = await request(config.url, { + // authentication: identityManager + // }) + // callback(response as FeatureServiceResult) + // } catch (err) { + // console.error(`Could not get ArcGIS layer info: ${err}`) + // // res.status(500).json({ message: 'Could not get ArcGIS layer info', error: err }) + // } + + // // this._httpClient.sendGetHandleResponse(url, this.parseFeatureService(url, callback)) + // } + + // /** + // * Parses the response from the feature service request and sends to the callback. + // * @param url The url to the arc feature layer. + // * @param callback The callback to call and send the feature service to. + // */ + // private parseFeatureService(url: string, callback: (featureService: FeatureServiceResult) => void) { + // return (chunk: any) => { + // this._console.log('Feature Service. url: ' + url + ', response: ' + chunk) + // try { + // const service = JSON.parse(chunk) as FeatureServiceResult + // callback(service) + // } catch (e) { + // this._console.error(e) + // } + // } + // } + + // /** + // * Queries an arc feature layer to get info on the layer. + // * @param url The url to the arc feature layer. + // * @param infoCallback Function to call once response has been received and parsed. + // */ + // async queryLayerInfo(config: FeatureServiceConfig, layerId: string | number, infoCallback: (layerInfo: LayerInfoResult) => void) { + // const httpClient = new HttpClient(this._console) + // try { + // const identityManager = await getIdentityManager(config, httpClient) + // const response = await request(config.url + '/' + layerId, { + // authentication: identityManager + // }) + // infoCallback(response as LayerInfoResult) + // } catch (err) { + // console.error(`Could not get ArcGIS layer info: ${err}`) + // // res.status(500).json({ message: 'Could not get ArcGIS layer info', error: err }) + // } + + // // this._httpClient.sendGetHandleResponse(url, this.parseLayerInfo(url, infoCallback)); + // } + + // /** + // * Parses the response from the request and sends the layer info to the callback. + // * @param url The url to the feature layer. + // * @param infoCallback The callback to call and send the layer info to. + // */ + // private parseLayerInfo(url: string, infoCallback: (layerInfo: LayerInfoResult) => void) { + // return (chunk: any) => { + // this._console.log('Query Layer. url: ' + url + ', response: ' + chunk) + // try { + // const layerInfo = JSON.parse(chunk) as LayerInfoResult + // infoCallback(layerInfo) + // } catch (e) { + // this._console.error(e) + // } + // } + // } } diff --git a/plugins/arcgis/service/src/FeatureServiceAdmin.ts b/plugins/arcgis/service/src/FeatureServiceAdmin.ts index 4539aa36a..d38fa7989 100644 --- a/plugins/arcgis/service/src/FeatureServiceAdmin.ts +++ b/plugins/arcgis/service/src/FeatureServiceAdmin.ts @@ -4,316 +4,291 @@ import { MageEvent, MageEventRepository } from '@ngageoint/mage.service/lib/enti import { Layer, Field } from "./AddLayersRequest" import { Form, FormField, FormFieldType, FormId } from '@ngageoint/mage.service/lib/entities/events/entities.events.forms' import { ObservationsTransformer } from "./ObservationsTransformer" -import { HttpClient } from './HttpClient' import { LayerInfoResult, LayerField } from "./LayerInfoResult" import FormData from 'form-data' +import { request } from '@esri/arcgis-rest-request' +import { ArcGISIdentityService } from "./ArcGISService" /** * Administers hosted feature services such as layer creation and updates. */ export class FeatureServiceAdmin { - - /** - * ArcGIS configuration. - */ - private _config: ArcGISPluginConfig - - /** - * Used to log to the console. - */ - private _console: Console - - /** - * Constructor. - * @param config The plugins configuration. - * @param console Used to log to the console. - */ - constructor(config: ArcGISPluginConfig, console: Console) { - this._config = config - this._console = console - } - - /** - * Create the layer - * @param service feature service - * @param featureLayer feature layer - * @param nextId next service layer id - * @param eventRepo event repository - * @returns layer id - */ - async createLayer(service: FeatureServiceConfig, featureLayer: FeatureLayerConfig, nextId: number, eventRepo: MageEventRepository): Promise { - - const layer = { type: 'Feature Layer' } as Layer - - const layerIdentifier = featureLayer.layer - const layerIdentifierNumber = Number(layerIdentifier) - if (isNaN(layerIdentifierNumber)) { - layer.name = String(layerIdentifier) - layer.id = nextId - } else { - layer.id = layerIdentifierNumber - } - - const events = await this.layerEvents(featureLayer, eventRepo) - - if (layer.name == null) { - layer.name = this.layerName(events) - } - - if (featureLayer.geometryType != null) { - layer.geometryType = featureLayer.geometryType - } else { - layer.geometryType = 'esriGeometryPoint' - } - - layer.fields = this.fields(events) - - // TODO What other layer properties are needed or required? - // https://developers.arcgis.com/rest/services-reference/online/add-to-definition-feature-service-.htm#GUID-63F2BD08-DCF4-485D-A3E6-C7116E17DDD8 - - this.create(service, layer) - - return layer.id - } - - /** - * Update the layer fields - * @param service feature service - * @param featureLayer feature layer - * @param layerInfo layer info - * @param eventRepo event repository - */ - async updateLayer(service: FeatureServiceConfig, featureLayer: FeatureLayerConfig, layerInfo: LayerInfoResult, eventRepo: MageEventRepository) { - - const events = await this.layerEvents(featureLayer, eventRepo) - - const eventFields = this.fields(events) - const layerFields = layerInfo.fields - - if (featureLayer.addFields) { - - const layerFieldSet = new Set() - for (const field of layerFields) { - layerFieldSet.add(field.name) - } - - const addFields = [] - for (const field of eventFields) { - if (!layerFieldSet.has(field.name)) { - addFields.push(field) - const layerField = {} as LayerField - layerField.name = field.name - layerField.editable = true - layerFields.push(layerField) - } - } - - if (addFields.length > 0) { - this.addFields(service, featureLayer, addFields) - } - - } - - if (featureLayer.deleteFields) { - - const eventFieldSet = new Set() - for (const field of eventFields) { - eventFieldSet.add(field.name) - } - - const deleteFields = [] - const remainingFields = [] - for (const field of layerFields) { - if (field.editable && !eventFieldSet.has(field.name)) { - deleteFields.push(field) - } else { - remainingFields.push(field) - } - } - - if (deleteFields.length > 0) { - layerInfo.fields = remainingFields - this.deleteFields(service, featureLayer, deleteFields) - } - - } - - } - - /** - * Get the layer events - * @param layer feature layer - * @param eventRepo event repository - * @returns layer events - */ - private async layerEvents(layer: FeatureLayerConfig, eventRepo: MageEventRepository): Promise { - - const layerEvents: Set = new Set() - if (layer.events != null) { - for (const layerEvent of layer.events) { - layerEvents.add(layerEvent) - } - } - - let mageEvents - if (layerEvents.size > 0) { - mageEvents = await eventRepo.findAll() - } else { - mageEvents = await eventRepo.findActiveEvents() - } - - const events: MageEvent[] = [] - for (const mageEvent of mageEvents) { - if (layerEvents.size == 0 || layerEvents.has(mageEvent.name) || layerEvents.has(mageEvent.id)) { - const event = await eventRepo.findById(mageEvent.id) - if (event != null) { - events.push(event) - } - } - } - - return events - } - - /** - * Create a layer name - * @param events layer events - * @returns layer name - */ - private layerName(events: MageEvent[]): string { - let layerName = '' - for (let i = 0; i < events.length; i++) { - if (i > 0) { - layerName += ', ' - } - layerName += events[i].name - } - return layerName - } - - /** - * Builder the layer fields - * @param events layer events - * @returns fields - */ - private fields(events: MageEvent[]): Field[] { - - const fields: Field[] = [] - - fields.push(this.createTextField(this._config.observationIdField, false)) - if (this._config.eventIdField != null) { - fields.push(this.createIntegerField(this._config.eventIdField, false)) - } - if (this._config.eventNameField != null) { - fields.push(this.createTextField(this._config.eventNameField, true)) - } - if (this._config.userIdField != null) { - fields.push(this.createTextField(this._config.userIdField, true)) - } - if (this._config.usernameField != null) { - fields.push(this.createTextField(this._config.usernameField, true)) - } - if (this._config.userDisplayNameField != null) { - fields.push(this.createTextField(this._config.userDisplayNameField, true)) - } - if (this._config.deviceIdField != null) { - fields.push(this.createTextField(this._config.deviceIdField, true)) - } - if (this._config.createdAtField != null) { - fields.push(this.createDateTimeField(this._config.createdAtField, true)) - } - if (this._config.lastModifiedField != null) { - fields.push(this.createDateTimeField(this._config.lastModifiedField, true)) - } - if (this._config.geometryType != null) { - fields.push(this.createTextField(this._config.geometryType, true)) - } - - const fieldNames = new Set() - for (const field of fields) { - fieldNames.add(field.name) - } - - this.eventsFields(events, fields, fieldNames) - - return fields - } - - /** - * Create a field - * @param name field name - * @param type form field type - * @param nullable nullable flag - * @param integer integer flag when numeric - * @returns field - */ - private createField(name: string, type: FormFieldType, nullable: boolean, integer?: boolean): Field { - let field = this.initField(type, integer) as Field - if (field != null) { - field.name = ObservationsTransformer.replaceSpaces(name) - field.alias = field.name - field.nullable = nullable - field.editable = true - } - return field - } - - /** - * Create a text field - * @param name field name - * @param nullable nullable flag - * @returns field - */ - private createTextField(name: string, nullable: boolean): Field { - return this.createField(name, FormFieldType.Text, nullable) - } - - /** - * Create a numeric field - * @param name field name - * @param nullable nullable flag - * @param integer integer flag - * @returns field - */ - private createNumericField(name: string, nullable: boolean, integer?: boolean): Field { - return this.createField(name, FormFieldType.Numeric, nullable, integer) - } - - /** - * Create an integer field - * @param name field name - * @param nullable nullable flag - * @returns field - */ - private createIntegerField(name: string, nullable: boolean): Field { - return this.createNumericField(name, nullable, true) - } - - /** - * Create a date time field - * @param name field name - * @param nullable nullable flag - * @returns field - */ - private createDateTimeField(name: string, nullable: boolean): Field { - return this.createField(name, FormFieldType.DateTime, nullable) - } - - /** - * Build fields from the layer events - * @param events layer events - * @param fields created fields - * @param fieldNames set of all field names - */ - private eventsFields(events: MageEvent[], fields: Field[], fieldNames: Set) { - - const forms = new Set() - - for (const event of events) { - this.eventFields(event, forms, fields, fieldNames) - } - - } + private _config: ArcGISPluginConfig + private _identityService: ArcGISIdentityService + private _console: Console + + constructor(config: ArcGISPluginConfig, identityService: ArcGISIdentityService, console: Console) { + this._config = config + this._identityService = identityService + this._console = console + } + + /** + * Create the layer + * @param service feature service + * @param featureLayer feature layer + * @param nextId next service layer id + * @param eventRepo event repository + * @returns layer id + */ + async createLayer(service: FeatureServiceConfig, featureLayer: FeatureLayerConfig, nextId: number, eventRepo: MageEventRepository): Promise { + const layer = { type: 'Feature Layer' } as Layer + + const layerIdentifier = featureLayer.layer + const layerIdentifierNumber = Number(layerIdentifier) + if (isNaN(layerIdentifierNumber)) { + layer.name = String(layerIdentifier) + layer.id = nextId + } else { + layer.id = layerIdentifierNumber + } + + const events = await this.layerEvents(featureLayer, eventRepo) + + if (layer.name == null) { + layer.name = this.layerName(events) + } + + if (featureLayer.geometryType != null) { + layer.geometryType = featureLayer.geometryType + } else { + layer.geometryType = 'esriGeometryPoint' + } + + layer.fields = this.fields(events) + + // TODO What other layer properties are needed or required? + // https://developers.arcgis.com/rest/services-reference/online/add-to-definition-feature-service-.htm#GUID-63F2BD08-DCF4-485D-A3E6-C7116E17DDD8 + + this.create(service, layer) + + return layer.id + } + + /** + * Update the layer fields + * @param service feature service + * @param featureLayer feature layer + * @param layerInfo layer info + * @param eventRepo event repository + */ + async updateLayer(service: FeatureServiceConfig, featureLayer: FeatureLayerConfig, layerInfo: LayerInfoResult, eventRepo: MageEventRepository) { + const events = await this.layerEvents(featureLayer, eventRepo) + + const eventFields = this.fields(events) + const layerFields = layerInfo.fields + + const layerFieldSet = new Set() + for (const field of layerFields) { + layerFieldSet.add(field.name) + } + + const addFields = [] + for (const field of eventFields) { + if (!layerFieldSet.has(field.name)) { + addFields.push(field) + const layerField = {} as LayerField + layerField.name = field.name + layerField.editable = true + layerFields.push(layerField) + } + } + + if (addFields.length > 0) { + await this.addFields(service, featureLayer, addFields) + } + + const eventFieldSet = new Set() + for (const field of eventFields) { + eventFieldSet.add(field.name) + } + + const deleteFields = [] + const remainingFields = [] + for (const field of layerFields) { + if (field.editable && !eventFieldSet.has(field.name)) { + deleteFields.push(field) + } else { + remainingFields.push(field) + } + } + + if (deleteFields.length > 0) { + layerInfo.fields = remainingFields + await this.deleteFields(service, featureLayer, deleteFields) + } + } + + /** + * Get the layer events + * @param layer feature layer + * @param eventRepo event repository + * @returns layer events + */ + private async layerEvents(layer: FeatureLayerConfig, eventRepo: MageEventRepository): Promise { + const layerEvents: Set = new Set() + if (layer.events != null) { + for (const layerEvent of layer.events) { + layerEvents.add(layerEvent) + } + } + + let mageEvents + if (layerEvents.size > 0) { + mageEvents = await eventRepo.findAll() + } else { + mageEvents = await eventRepo.findActiveEvents() + } + + const events: MageEvent[] = [] + for (const mageEvent of mageEvents) { + if (layerEvents.size == 0 || layerEvents.has(mageEvent.name) || layerEvents.has(mageEvent.id)) { + const event = await eventRepo.findById(mageEvent.id) + if (event != null) { + events.push(event) + } + } + } + + return events + } + + /** + * Create a layer name + * @param events layer events + * @returns layer name + */ + private layerName(events: MageEvent[]): string { + let layerName = '' + for (let i = 0; i < events.length; i++) { + if (i > 0) { + layerName += ', ' + } + layerName += events[i].name + } + return layerName + } + + /** + * Builder the layer fields + * @param events layer events + * @returns fields + */ + private fields(events: MageEvent[]): Field[] { + const fields: Field[] = [] + + fields.push(this.createTextField(this._config.observationIdField, false)) + if (this._config.eventIdField != null) { + fields.push(this.createIntegerField(this._config.eventIdField, false)) + } + if (this._config.eventNameField != null) { + fields.push(this.createTextField(this._config.eventNameField, true)) + } + if (this._config.userIdField != null) { + fields.push(this.createTextField(this._config.userIdField, true)) + } + if (this._config.usernameField != null) { + fields.push(this.createTextField(this._config.usernameField, true)) + } + if (this._config.userDisplayNameField != null) { + fields.push(this.createTextField(this._config.userDisplayNameField, true)) + } + if (this._config.deviceIdField != null) { + fields.push(this.createTextField(this._config.deviceIdField, true)) + } + if (this._config.createdAtField != null) { + fields.push(this.createDateTimeField(this._config.createdAtField, true)) + } + if (this._config.lastModifiedField != null) { + fields.push(this.createDateTimeField(this._config.lastModifiedField, true)) + } + if (this._config.geometryType != null) { + fields.push(this.createTextField(this._config.geometryType, true)) + } + + const fieldNames = new Set() + for (const field of fields) { + fieldNames.add(field.name) + } + + this.eventsFields(events, fields, fieldNames) + + return fields + } + + /** + * Create a field + * @param name field name + * @param type form field type + * @param nullable nullable flag + * @param integer integer flag when numeric + * @returns field + */ + private createField(name: string, type: FormFieldType, nullable: boolean, integer?: boolean): Field { + let field = this.initField(type, integer) as Field + if (field != null) { + field.name = ObservationsTransformer.replaceSpaces(name) + field.alias = field.name + field.nullable = nullable + field.editable = true + } + return field + } + + /** + * Create a text field + * @param name field name + * @param nullable nullable flag + * @returns field + */ + private createTextField(name: string, nullable: boolean): Field { + return this.createField(name, FormFieldType.Text, nullable) + } + + /** + * Create a numeric field + * @param name field name + * @param nullable nullable flag + * @param integer integer flag + * @returns field + */ + private createNumericField(name: string, nullable: boolean, integer?: boolean): Field { + return this.createField(name, FormFieldType.Numeric, nullable, integer) + } + + /** + * Create an integer field + * @param name field name + * @param nullable nullable flag + * @returns field + */ + private createIntegerField(name: string, nullable: boolean): Field { + return this.createNumericField(name, nullable, true) + } + + /** + * Create a date time field + * @param name field name + * @param nullable nullable flag + * @returns field + */ + private createDateTimeField(name: string, nullable: boolean): Field { + return this.createField(name, FormFieldType.DateTime, nullable) + } + + /** + * Build fields from the layer events + * @param events layer events + * @param fields created fields + * @param fieldNames set of all field names + */ + private eventsFields(events: MageEvent[], fields: Field[], fieldNames: Set) { + const forms = new Set() + + for (const event of events) { + this.eventFields(event, forms, fields, fieldNames) + } + } /** * Build fields from the layer event @@ -322,210 +297,189 @@ export class FeatureServiceAdmin { * @param fields created fields * @param fieldNames set of all field names */ - private eventFields(event: MageEvent, forms: Set, fields: Field[], fieldNames: Set) { - - for (const form of event.activeForms) { - - if (!forms.has(form.id)) { - - forms.add(form.id) - - for (const formField of form.fields) { - if (formField.archived == null || !formField.archived) { - this.createFormField(form, formField, fields, fieldNames) - } - } - - } - } - - } - - /** - * Build a field from the form field - * @param form form - * @param formField form field - * @param fields created fields - * @param fieldNames set of all field names - */ - private createFormField(form: Form, formField: FormField, fields: Field[], fieldNames: Set) { - - const field = this.initField(formField.type) - - if (field != null) { - - let name = ObservationsTransformer.replaceSpaces(formField.title) - - if (fieldNames.has(name)) { - name = form.name + '_' + name - } - fieldNames.add(name) - - field.name = name - field.alias = field.name - field.nullable = !formField.required - field.editable = true - field.defaultValue = formField.value - - fields.push(field) - } - - } - - /** - * Initialize a field by type - * @param type form field type - * @param integer numeric integer field type - * @return field or null - */ - private initField(type: FormFieldType, integer?: boolean): Field | null { - - let field = {} as Field - - switch (type) { - case FormFieldType.CheckBox: - case FormFieldType.Dropdown: - case FormFieldType.Email: - case FormFieldType.MultiSelectDropdown: - case FormFieldType.Password: - case FormFieldType.Radio: - case FormFieldType.Text: - field.type = 'esriFieldTypeString' - field.actualType = 'nvarchar' - field.sqlType = 'sqlTypeNVarchar' - field.length = this._config.textFieldLength - break; - case FormFieldType.TextArea: - field.type = 'esriFieldTypeString' - field.actualType = 'nvarchar' - field.sqlType = 'sqlTypeNVarchar' - field.length = this._config.textAreaFieldLength - break; - case FormFieldType.DateTime: - field.type = 'esriFieldTypeDate' - field.sqlType = 'sqlTypeOther' - field.length = 10 - break; - case FormFieldType.Numeric: - if (integer) { - field.type = 'esriFieldTypeInteger' - field.actualType = 'int' - field.sqlType = 'sqlTypeInteger' - } else { - field.type = 'esriFieldTypeDouble' - field.actualType = 'float' - field.sqlType = 'sqlTypeFloat' - } - break; - case FormFieldType.Geometry: - case FormFieldType.Attachment: - case FormFieldType.Hidden: - default: - break - } - - return field.type != null ? field : null - } - - /** - * Create the layer - * @param service feature service - * @param layer layer - */ - private create(service: FeatureServiceConfig, layer: Layer) { - - const httpClient = this.httpClient(service) - const url = this.adminUrl(service) + 'addToDefinition' - - this._console.info('ArcGIS feature service addToDefinition (create layer) url ' + url) - - const form = new FormData() - form.append('addToDefinition', JSON.stringify(layer)) - - httpClient.sendPostForm(url, form) - - } - - /** - * Add fields to the layer - * @param service feature service - * @param featureLayer feature layer - * @param fields fields to add - */ - private addFields(service: FeatureServiceConfig, featureLayer: FeatureLayerConfig, fields: Field[]) { - - const layer = {} as Layer - layer.fields = fields - - const httpClient = this.httpClient(service) - const url = this.adminUrl(service) + featureLayer.layer.toString() + '/addToDefinition' - - this._console.info('ArcGIS feature layer addToDefinition (add fields) url ' + url) - - const form = new FormData() - form.append('addToDefinition', JSON.stringify(layer)) - - httpClient.sendPostForm(url, form) - - } - - /** - * Delete fields from the layer - * @param service feature service - * @param featureLayer feature layer - * @param fields fields to delete - */ - private deleteFields(service: FeatureServiceConfig, featureLayer: FeatureLayerConfig, fields: LayerField[]) { - - const deleteFields = [] - for (const layerField of fields) { - const field = {} as Field - field.name = layerField.name - deleteFields.push(field) - } - - const layer = {} as Layer - layer.fields = deleteFields - - const httpClient = this.httpClient(service) - const url = this.adminUrl(service) + featureLayer.layer.toString() + '/deleteFromDefinition' - - this._console.info('ArcGIS feature layer deleteFromDefinition (delete fields) url ' + url) - - const form = new FormData() - form.append('deleteFromDefinition', JSON.stringify(layer)) - - httpClient.sendPostForm(url, form) - - } - - /** - * Get the administration url - * @param service feature service - * @returns url - */ - private adminUrl(service: FeatureServiceConfig): String { - let url = service.adminUrl - if (url == null) { - url = service.url.replace('/services/', '/admin/services/') - } - if (!url.endsWith('/')) { - url += '/' - } - return url - } - - /** - * Get a HTTP Client with administration token - * @param service feature service - * @returns http client - */ - private httpClient(service: FeatureServiceConfig): HttpClient { - let token = service.adminToken - if (token == null) { - token = service.auth?.type == 'token' ? service.auth.token : "" - } - return new HttpClient(console, token) - } - -} + private eventFields(event: MageEvent, forms: Set, fields: Field[], fieldNames: Set) { + for (const form of event.activeForms) { + if (!forms.has(form.id)) { + forms.add(form.id) + + for (const formField of form.fields) { + if (formField.archived == null || !formField.archived) { + this.createFormField(form, formField, fields, fieldNames) + } + } + } + } + } + + /** + * Build a field from the form field + * @param form form + * @param formField form field + * @param fields created fields + * @param fieldNames set of all field names + */ + private createFormField(form: Form, formField: FormField, fields: Field[], fieldNames: Set) { + const field = this.initField(formField.type) + + if (field != null) { + const sanitizedName = ObservationsTransformer.replaceSpaces(formField.title) + const sanitizedFormName = ObservationsTransformer.replaceSpaces(form.name) + const name = `${sanitizedFormName}_${sanitizedName}`.toLowerCase() + + fieldNames.add(name) + + field.name = name + field.alias = field.name + field.nullable = !formField.required + field.editable = true + field.defaultValue = formField.value + + fields.push(field) + } + } + + /** + * Initialize a field by type + * @param type form field type + * @param integer numeric integer field type + * @return field or null + */ + private initField(type: FormFieldType, integer?: boolean): Field | null { + let field = {} as Field + + switch (type) { + case FormFieldType.CheckBox: + case FormFieldType.Dropdown: + case FormFieldType.Email: + case FormFieldType.MultiSelectDropdown: + case FormFieldType.Password: + case FormFieldType.Radio: + case FormFieldType.Text: + field.type = 'esriFieldTypeString' + field.actualType = 'nvarchar' + field.sqlType = 'sqlTypeNVarchar' + field.length = this._config.textFieldLength + break; + case FormFieldType.TextArea: + field.type = 'esriFieldTypeString' + field.actualType = 'nvarchar' + field.sqlType = 'sqlTypeNVarchar' + field.length = this._config.textAreaFieldLength + break; + case FormFieldType.DateTime: + field.type = 'esriFieldTypeDate' + field.sqlType = 'sqlTypeOther' + field.length = 10 + break; + case FormFieldType.Numeric: + if (integer) { + field.type = 'esriFieldTypeInteger' + field.actualType = 'int' + field.sqlType = 'sqlTypeInteger' + } else { + field.type = 'esriFieldTypeDouble' + field.actualType = 'float' + field.sqlType = 'sqlTypeFloat' + } + break; + case FormFieldType.Geometry: + case FormFieldType.Attachment: + case FormFieldType.Hidden: + default: + break + } + + return field.type != null ? field : null + } + + /** + * Create the layer + * @param service feature service + * @param layer layer + */ + private async create(service: FeatureServiceConfig, layer: Layer) { + const url = this.adminUrl(service) + 'addToDefinition' + + this._console.info('ArcGIS feature service addToDefinition (create layer) url ' + url) + + const form = new FormData() + form.append('addToDefinition', JSON.stringify(layer)) + + const identityManager = await this._identityService.signin(service) + await request(url, { + authentication: identityManager, + httpMethod: 'POST', + params: form + }).catch((error) => this._console.error('FeatureServiceAdmin create() error ' + error)); + } + + /** + * Add fields to the layer + * @param service feature service + * @param featureLayer feature layer + * @param fields fields to add + */ + private async addFields(service: FeatureServiceConfig, featureLayer: FeatureLayerConfig, fields: Field[]) { + const layer = { fields: fields} as Layer + + const url = this.adminUrl(service) + featureLayer.layer.toString() + '/addToDefinition' + + this._console.info('ArcGIS feature layer addToDefinition (add fields) url ' + url) + + const identityManager = await this._identityService.signin(service) + await request(url, { + authentication: identityManager, + params: { + addToDefinition: layer, + f: "json" + } + }).catch((error) => { + console.log('Error: ' + error) + }); + } + + /** + * Delete fields from the layer + * @param service feature service + * @param featureLayer feature layer + * @param fields fields to delete + */ + private async deleteFields(service: FeatureServiceConfig, featureLayer: FeatureLayerConfig, fields: LayerField[]) { + const deleteFields = [] + for (const layerField of fields) { + const field = {} as Field + field.name = layerField.name + deleteFields.push(field) + } + + const layer = {} as Layer + layer.fields = deleteFields + + const url = this.adminUrl(service) + featureLayer.layer.toString() + '/deleteFromDefinition' + + this._console.info('ArcGIS feature layer deleteFromDefinition (delete fields) url ' + url) + + const identityManager = await this._identityService.signin(service) + request(url, { + authentication: identityManager, + httpMethod: 'POST', + params: { + deleteFromDefinition: layer + } + }).catch((error) => this._console.error('FeatureServiceAdmin deleteFields() error ' + error)); + } + + /** + * Get the administration url + * @param service feature service + * @returns url + */ + private adminUrl(service: FeatureServiceConfig): String { + let url = service.url.replace('/services/', '/admin/services/') + if (!url.endsWith('/')) { + url += '/' + } + + return url + } +} \ No newline at end of file diff --git a/plugins/arcgis/service/src/HttpClient.ts b/plugins/arcgis/service/src/HttpClient.ts deleted file mode 100644 index 246fd9a51..000000000 --- a/plugins/arcgis/service/src/HttpClient.ts +++ /dev/null @@ -1,228 +0,0 @@ -import https from 'https'; -import FormData from 'form-data'; - -/** - * Makes Http calls on specified urls. - */ -export class HttpClient { - - /** - * Used to log to the console. - */ - private _console: Console; - - /** - * The access token - */ - private _token?: string - - /** - * Constructor. - * @param console Used to log to the console. - * @param token The access token. - */ - constructor(console: Console, token?: string) { - this._console = console - this._token = token - } - - /** - * Sends a post request to the specified url with the specified data. - * @param url The url to send a post request to. - * @param formData The data to put in the post. - */ - sendPost(url: string, formData: string) { - const console = this._console - this.sendPostHandleResponse(url, formData, function (chunk) { - console.log('Response: ' + chunk); - }) - } - - /** - * Sends a post request to the specified url with the specified data. - * @param url The url to send a post request to. - * @param formData The data to put in the post. - * @param response The post response handler function. - */ - sendPostHandleResponse(url: string, formData: string, response: (chunk: any) => void) { - const aUrl = new URL(url); - - formData += this.tokenParam() - formData += this.jsonFormatParam() - - var post_options = { - host: aUrl.host, - port: aUrl.port, - path: aUrl.pathname, - method: 'POST', - headers: { - 'content-type': 'application/x-www-form-urlencoded', - 'content-length': Buffer.byteLength(formData), - 'accept': 'application/json' - } - }; - - // Set up the request - var post_req = https.request(post_options, function (res) { - res.setEncoding('utf8'); - res.on('data', response); - }); - - // post the data - post_req.write(formData); - post_req.end(); - } - - /** - * Sends a post request to the specified url with the specified data. - * @param url The url to send a post request to. - * @param form The data to put in the post. - */ - sendPostForm(url: string, form: FormData) { - const console = this._console - this.sendPostFormHandleResponse(url, form, function (chunk) { - console.log('Response: ' + chunk) - }) - } - - /** - * Sends a post request to the specified url with the specified data. - * @param url The url to send a post request to. - * @param form The data to put in the post. - * @param response The post response handler function. - */ - sendPostFormHandleResponse(url: string, form: FormData, response: (chunk: any) => void) { - const aUrl = new URL(url) - - if (this._token != null) { - form.append('token', this._token) - } - form.append('f', 'json') - - var post_options = { - host: aUrl.host, - port: aUrl.port, - path: aUrl.pathname, - method: 'POST', - headers: form.getHeaders() - }; - - // Set up the request - var post_req = https.request(post_options, function (res) { - res.setEncoding('utf8') - res.on('data', response) - }); - - // post the data - form.pipe(post_req) - } - - /** - * Sends a get request to the specified url. - * @param url The url of the get request. - */ - async sendGet(url: string): Promise { - const console = this._console - return new Promise((resolve, reject) => { - try { - this.sendGetHandleResponse(url, function (chunk) { - console.log('Response: ' + chunk); - resolve(JSON.parse(chunk)); - }) - } catch (error) { - reject(error) - } - }); - } - - /** - * Sends a get request to the specified url. - * @param url The url of the get request. - * @param response The get response handler function. - */ - sendGetHandleResponse(url: string, response: (chunk: any) => void) { - - url += this.tokenParam(url) - url += this.jsonFormatParam(url) - - try { - const aUrl = new URL(url) - - var options = { - host: aUrl.host, - port: aUrl.port, - path: aUrl.pathname + '?' + aUrl.searchParams, - method: 'GET', - headers: { - 'accept': 'application/json' - } - }; - - // Set up the request - var get_req = https.request(options, function (res) { - let data = ''; - res.setEncoding('utf8'); - res.on('data', (chunk: string): void => {data += chunk;}); - res.on('end', (): void =>{response(data);}); - }); - - get_req.on('error', function(error) { - console.log('Error for ' + url + ' ' + error); - }); - - // get the data - get_req.end(); - } catch (e) { - if (e instanceof TypeError) { - console.log('Error for ' + url + ' ' + e) - response('{}') - } else { - throw e;//cause we dont know what it is or we want to only handle TypeError - } - } - } - - /** - * Get the token parameter - * @param url URL the parameter will be added to - * @returns token param - */ - private tokenParam(url?: string): string { - let token = '' - if (this._token != null) { - token += this.paramSeparator(url) - token += "token=" + this._token - } - return token - } - - /** - * Get the JSON format parameter - * @param url URL the parameter will be added to - * @returns json format parameter - */ - private jsonFormatParam(url?: string): string { - return this.paramSeparator(url) + "f=json" - } - - /** - * Get the parameter separator - * @param url URL a separator will be added to - * @returns parameter separator - */ - private paramSeparator(url?: string): string { - let separator = '' - if (url != null && url.length > 0) { - const index = url.indexOf('?') - if (index == -1) { - separator = '?' - } else if (index < url.length - 1){ - separator = '&' - } - } else { - separator = '&' - } - return separator - } - -} \ No newline at end of file diff --git a/plugins/arcgis/service/src/LayerInfo.ts b/plugins/arcgis/service/src/LayerInfo.ts index 931937cf4..4918d7153 100644 --- a/plugins/arcgis/service/src/LayerInfo.ts +++ b/plugins/arcgis/service/src/LayerInfo.ts @@ -37,9 +37,8 @@ export class LayerInfo { * @param layerInfo The layer info. * @param token The access token. */ - constructor(url: string, events: string[], layerInfo: LayerInfoResult, token?: string) { + constructor(url: string, events: string[], layerInfo: LayerInfoResult) { this.url = url - this.token = token if (events != undefined && events != null && events.length == 0) { this.events.add('nothing to sync') } diff --git a/plugins/arcgis/service/src/ObservationBinner.ts b/plugins/arcgis/service/src/ObservationBinner.ts index 2b7794ff2..ca2477552 100644 --- a/plugins/arcgis/service/src/ObservationBinner.ts +++ b/plugins/arcgis/service/src/ObservationBinner.ts @@ -89,9 +89,7 @@ export class ObservationBinner { const bins = new ObservationBins(); for (const arcObservation of observations.observations) { - const arcObject = arcObservation.object - if (observations.firstRun - || arcObservation.lastModified != arcObservation.createdAt) { + if (arcObservation.lastModified != arcObservation.createdAt) { bins.updates.add(arcObservation); } else if (!this._addedObs.has(arcObservation.id)) { bins.adds.add(arcObservation); diff --git a/plugins/arcgis/service/src/ObservationProcessor.ts b/plugins/arcgis/service/src/ObservationProcessor.ts index e6c526b64..c4ffed54a 100644 --- a/plugins/arcgis/service/src/ObservationProcessor.ts +++ b/plugins/arcgis/service/src/ObservationProcessor.ts @@ -14,9 +14,11 @@ import { EventTransform } from './EventTransform'; import { GeometryChangedHandler } from './GeometryChangedHandler'; import { EventDeletionHandler } from './EventDeletionHandler'; import { EventLayerProcessorOrganizer } from './EventLayerProcessorOrganizer'; -import { FeatureServiceConfig, FeatureLayerConfig, AuthType, OAuthAuthConfig } from "./ArcGISConfig" +import { FeatureServiceConfig, FeatureLayerConfig } from "./ArcGISConfig" import { PluginStateRepository } from '@ngageoint/mage.service/lib/plugins.api' import { FeatureServiceAdmin } from './FeatureServiceAdmin'; +import { request } from '@esri/arcgis-rest-request'; +import { ArcGISIdentityService } from './ArcGISService'; /** * Class that wakes up at a certain configured interval and processes any new observations that can be @@ -24,409 +26,398 @@ import { FeatureServiceAdmin } from './FeatureServiceAdmin'; */ export class ObservationProcessor { - /** - * True if the processor is currently active, false otherwise. - */ - private _isRunning = false; - - /** - * The next timeout, use this to cancel the next one if the processor is stopped. - */ - private _nextTimeout: NodeJS.Timeout | undefined; - - /** - * Used to get all the active events. - */ - private _eventRepo: MageEventRepository; - - /** - * Used to get new observations. - */ - private _obsRepos: ObservationRepositoryForEvent; - - /** - * Used to get user information. - */ - private _userRepo: UserRepository; - - /** - * Used to log to the console. - */ - private _console: Console; - - /** - * Used to convert observations to json string that can be sent to an arcgis server. - */ - private _transformer: ObservationsTransformer; - - /** - * Contains the different feature layers to send observations too. - */ - private _stateRepo: PluginStateRepository; - - /** - * The previous plugins configuration JSON. - */ - private _previousConfig?: string; - - /** - * Sends observations to a single feature layer. - */ - private _layerProcessors: FeatureLayerProcessor[] = []; - - /** - * True if this is a first run at updating arc feature layers. If so we need to make sure the layers are - * all up to date. - */ - private _firstRun: boolean; - - /** - * Handles removing observation from previous layers when an observation geometry changes. - */ - private _geometryChangeHandler: GeometryChangedHandler; - - /** - * Handles removing observations when an event is deleted. - */ - private _eventDeletionHandler: EventDeletionHandler; - - /** - * Maps the events to the processor they are synching data for. - */ - private _organizer: EventLayerProcessorOrganizer; - - /** - * Constructor. - * @param stateRepo The plugins configuration. - * @param eventRepo Used to get all the active events. - * @param obsRepo Used to get new observations. - * @param userRepo Used to get user information. - * @param console Used to log to the console. - */ - constructor(stateRepo: PluginStateRepository, eventRepo: MageEventRepository, obsRepos: ObservationRepositoryForEvent, userRepo: UserRepository, console: Console) { - this._stateRepo = stateRepo; - this._eventRepo = eventRepo; - this._obsRepos = obsRepos; - this._userRepo = userRepo; - this._console = console; - this._firstRun = true; - this._organizer = new EventLayerProcessorOrganizer(); - this._transformer = new ObservationsTransformer(defaultArcGISPluginConfig, console); - this._geometryChangeHandler = new GeometryChangedHandler(this._transformer); - this._eventDeletionHandler = new EventDeletionHandler(this._console, defaultArcGISPluginConfig); - } - - /** - * Gets the current configuration from the database. - * @returns The current configuration from the database. - */ - public async safeGetConfig(): Promise { - const state = await this._stateRepo.get(); - if (!state) return await this._stateRepo.put(defaultArcGISPluginConfig); - return await this._stateRepo.get().then((state) => state ? state : this._stateRepo.put(defaultArcGISPluginConfig)); - } - - /** - * Puts a new confguration in the state repo. - * @param newConfig The new config to put into the state repo. - */ - public async putConfig(newConfig: ArcGISPluginConfig): Promise { - return await this._stateRepo.put(newConfig); - } - - /** - * Updates the confguration in the state repo. - * @param newConfig The new config to put into the state repo. - */ - public async patchConfig(newConfig: ArcGISPluginConfig): Promise { - return await this._stateRepo.patch(newConfig); - } - - /** - * Gets the current configuration and updates the processor if needed - * @returns The current configuration from the database. - */ - private async updateConfig(): Promise { - const config = await this.safeGetConfig() - const configJson = JSON.stringify(config) - if (this._previousConfig == null || this._previousConfig != configJson) { - this._transformer = new ObservationsTransformer(config, console); - this._geometryChangeHandler = new GeometryChangedHandler(this._transformer); - this._eventDeletionHandler = new EventDeletionHandler(this._console, config); - this._layerProcessors = []; - this.getFeatureServiceLayers(config); - this._previousConfig = configJson - this._firstRun = true; - } - return config - } - - /** - * Starts the processor. - */ - async start() { - this._isRunning = true; - this._firstRun = true; - this.processAndScheduleNext(); - } - - /** - * Stops the processor. - */ - stop() { - this._isRunning = false; - clearTimeout(this._nextTimeout); - } - - /** - * Gets information on all the configured features service layers. - * @param config The plugins configuration. - */ - private getFeatureServiceLayers(config: ArcGISPluginConfig) { - // TODO: What is the impact of what this is doing? Do we need to account for usernamePassword auth type services? - for (const service of config.featureServices) { - - const services: FeatureServiceConfig[] = [] - - if (service.auth?.type !== AuthType.Token || service.auth?.token == null) { - const tokenServices = new Map() - const nonTokenLayers = [] - for (const layer of service.layers) { - if (layer.token != null) { - let serv = tokenServices.get(layer.token) - if (serv == null) { - serv = { url: service.url, token: layer.token, layers: [] } - tokenServices.set(layer.token, serv) - services.push(serv) - } - serv.layers.push(layer) - } else { - nonTokenLayers.push(layer) - } - } - if (services.length > 0) { - service.layers = nonTokenLayers - } - } - - if (service.layers.length > 0) { - services.push(service) - } - - for (const serv of services) { - const featureService = new FeatureService(console, (serv.auth?.type === AuthType.Token && serv.auth?.token != null) ? serv.auth.token : '') - featureService.queryFeatureService(serv.url, (featureServiceResult: FeatureServiceResult) => this.handleFeatureService(featureServiceResult, serv, config)) - } - } - } - - /** - * Called when information on a feature service is returned from an arc server. - * @param featureService The feature service. - * @param featureServiceConfig The feature service config. - * @param config The plugin configuration. - */ - private async handleFeatureService(featureService: FeatureServiceResult, featureServiceConfig: FeatureServiceConfig, config: ArcGISPluginConfig) { - - if (featureService.layers != null) { - - const serviceLayers = new Map() - const admin = new FeatureServiceAdmin(config, this._console) - - let maxId = -1 - for (const layer of featureService.layers) { - serviceLayers.set(layer.id, layer) - serviceLayers.set(layer.name, layer) - maxId = Math.max(maxId, layer.id) - } - - for (const featureLayer of featureServiceConfig.layers) { - - if (featureLayer.token == null) { - featureLayer.token = featureServiceConfig.auth?.type == AuthType.Token ? featureServiceConfig.auth.token : "" - } - - const eventNames: string[] = [] - const events = featureLayer.events - if (events != null) { - for (const event of events) { - const eventId = Number(event); - if (isNaN(eventId)) { - eventNames.push(String(event)); - } else { - const mageEvent = await this._eventRepo.findById(eventId) - if (mageEvent != null) { - eventNames.push(mageEvent.name); - } - } - } - } - if (eventNames.length > 0) { - featureLayer.events = eventNames - } - - const layer = serviceLayers.get(featureLayer.layer) - - let layerId = undefined - if (layer != null) { - layerId = layer.id - } else if (featureServiceConfig.createLayers) { - layerId = await admin.createLayer(featureServiceConfig, featureLayer, maxId + 1, this._eventRepo) - maxId = Math.max(maxId, layerId) - } - - if (layerId != null) { - featureLayer.layer = layerId - const featureService = new FeatureService(console, featureLayer.token) - const url = featureServiceConfig.url + '/' + layerId - featureService.queryLayerInfo(url, (layerInfo: LayerInfoResult) => this.handleLayerInfo(url, featureServiceConfig, featureLayer, layerInfo, config)) - } - - } - - } - } - - /** - * Called when information on a feature layer is returned from an arc server. - * @param url The layer url. - * @param featureServiceConfig The feature service config. - * @param featureLayer The feature layer configuration. - * @param layerInfo The information on a layer. - * @param config The plugins configuration. - */ - private async handleLayerInfo(url: string, featureServiceConfig: FeatureServiceConfig, featureLayer: FeatureLayerConfig, layerInfo: LayerInfoResult, config: ArcGISPluginConfig) { - if (layerInfo.geometryType != null) { - const events = featureLayer.events as string[] - if (featureLayer.addFields || featureLayer.deleteFields) { - const admin = new FeatureServiceAdmin(config, this._console) - await admin.updateLayer(featureServiceConfig, featureLayer, layerInfo, this._eventRepo) - } - const info = new LayerInfo(url, events, layerInfo, featureLayer.token) - const layerProcessor = new FeatureLayerProcessor(info, config, this._console); - this._layerProcessors.push(layerProcessor); - clearTimeout(this._nextTimeout); - this.scheduleNext(config); - } - } - - /** - * Processes any new observations and then schedules its next run if it hasn't been stopped. - */ - private async processAndScheduleNext() { - const config = await this.updateConfig(); - if (this._isRunning) { - if (config.enabled && this._layerProcessors.length > 0) { - this._console.info('ArcGIS plugin checking for any pending updates or adds'); - for (const layerProcessor of this._layerProcessors) { - layerProcessor.processPendingUpdates(); - } - this._console.info('ArcGIS plugin processing new observations...'); - const activeEvents = await this._eventRepo.findActiveEvents(); - this._eventDeletionHandler.checkForEventDeletion(activeEvents, this._layerProcessors, this._firstRun); - const eventsToProcessors = this._organizer.organize(activeEvents, this._layerProcessors); - const nextQueryTime = Date.now(); - for (const pair of eventsToProcessors) { - this._console.info('ArcGIS getting newest observations for event ' + pair.event.name); - const obsRepo = await this._obsRepos(pair.event.id); - const pagingSettings = { - pageSize: config.batchSize, - pageIndex: 0, - includeTotalCount: true - } - let morePages = true; - let numberLeft = 0; - while (morePages) { - numberLeft = await this.queryAndSend(config, pair.featureLayerProcessors, obsRepo, pagingSettings, numberLeft); - morePages = numberLeft > 0; - } - } - - for (const layerProcessor of this._layerProcessors) { - layerProcessor.lastTimeStamp = nextQueryTime; - } - - this._firstRun = false; - } - this.scheduleNext(config); - } - } - - private scheduleNext(config: ArcGISPluginConfig) { - if (this._isRunning) { - let interval = config.intervalSeconds; - if (this._firstRun && config.featureServices.length > 0) { - interval = config.startupIntervalSeconds; - } else { - for (const layerProcessor of this._layerProcessors) { - if (layerProcessor.hasPendingUpdates()) { - interval = config.updateIntervalSeconds; - break; - } - } - } - this._nextTimeout = setTimeout(() => { this.processAndScheduleNext() }, interval * 1000); - } - } - - /** - * Queries for new observations and sends them to any configured arc servers. - * @param config The plugin configuration. - * @param layerProcessors The layer processors to use when processing arc objects. - * @param obsRepo The observation repo for an event. - * @param pagingSettings Current paging settings. - * @param numberLeft The number of observations left to query and send to arc. - * @returns The number of observations still needing to be queried and sent to arc. - */ - private async queryAndSend(config: ArcGISPluginConfig, layerProcessors: FeatureLayerProcessor[], obsRepo: EventScopedObservationRepository, pagingSettings: PagingParameters, numberLeft: number): Promise { - let newNumberLeft = numberLeft; - - let queryTime = -1; - for (const layerProcessor of layerProcessors) { - if (queryTime == -1 || layerProcessor.lastTimeStamp < queryTime) { - queryTime = layerProcessor.lastTimeStamp; - } - } - - let latestObs = await obsRepo.findLastModifiedAfter(queryTime, pagingSettings); - if (latestObs != null && latestObs.totalCount != null && latestObs.totalCount > 0) { - if (pagingSettings.pageIndex == 0) { - this._console.info('ArcGIS newest observation count ' + latestObs.totalCount); - newNumberLeft = latestObs.totalCount; - } - const observations = latestObs.items - const mageEvent = await this._eventRepo.findById(obsRepo.eventScope) - const eventTransform = new EventTransform(config, mageEvent) - const arcObjects = new ArcObjects() - this._geometryChangeHandler.checkForGeometryChange(observations, arcObjects, layerProcessors, this._firstRun); - for (let i = 0; i < observations.length; i++) { - const observation = observations[i] - let deletion = false - if (observation.states.length > 0) { - deletion = observation.states[0].name.startsWith('archive') - } - if (deletion) { - const arcObservation = this._transformer.createObservation(observation) - arcObjects.deletions.push(arcObservation) - } else { - let user = null - if (observation.userId != null) { - user = await this._userRepo.findById(observation.userId) - } - const arcObservation = this._transformer.transform(observation, eventTransform, user) - arcObjects.add(arcObservation) - } - } - arcObjects.firstRun = this._firstRun; - for (const layerProcessor of layerProcessors) { - layerProcessor.processArcObjects(JSON.parse(JSON.stringify(arcObjects))); - } - newNumberLeft -= latestObs.items.length; - pagingSettings.pageIndex++; - } else { - this._console.info('ArcGIS no new observations') - } - - return newNumberLeft; - } + /** + * True if the processor is currently active, false otherwise. + */ + private _isRunning = false; + + /** + * The next timeout, use this to cancel the next one if the processor is stopped. + */ + private _nextTimeout: NodeJS.Timeout | undefined; + + /** + * Used to get all the active events. + */ + private _eventRepo: MageEventRepository; + + /** + * Used to get new observations. + */ + private _obsRepos: ObservationRepositoryForEvent; + + /** + * Used to get user information. + */ + private _userRepo: UserRepository; + + /** + * Used to manager ArcGIS user identities + */ + private _identityService: ArcGISIdentityService + + /** + * Used to log to the console. + */ + private _console: Console; + + /** + * Used to convert observations to json string that can be sent to an arcgis server. + */ + private _transformer: ObservationsTransformer; + + /** + * Contains the different feature layers to send observations too. + */ + private _stateRepo: PluginStateRepository; + + /** + * The previous plugins configuration JSON. + */ + private _previousConfig?: string; + + /** + * Sends observations to a single feature layer. + */ + private _layerProcessors: FeatureLayerProcessor[] = []; + + /** + * True if this is a first run at updating arc feature layers. If so we need to make sure the layers are + * all up to date. + */ + private _firstRun: boolean; + + /** + * Handles removing observation from previous layers when an observation geometry changes. + */ + private _geometryChangeHandler: GeometryChangedHandler; + + /** + * Handles removing observations when an event is deleted. + */ + private _eventDeletionHandler: EventDeletionHandler; + + /** + * Maps the events to the processor they are synching data for. + */ + private _organizer: EventLayerProcessorOrganizer; + + /** + * Constructor. + * @param stateRepo The plugins configuration. + * @param eventRepo Used to get all the active events. + * @param obsRepo Used to get new observations. + * @param userRepo Used to get user information. + * @param console Used to log to the console. + */ + constructor( + stateRepo: PluginStateRepository, + eventRepo: MageEventRepository, + obsRepos: ObservationRepositoryForEvent, + userRepo: UserRepository, + identityService: ArcGISIdentityService, + console: Console + ) { + this._stateRepo = stateRepo; + this._eventRepo = eventRepo; + this._obsRepos = obsRepos; + this._userRepo = userRepo; + this._identityService = identityService + this._console = console; + this._firstRun = true; + this._organizer = new EventLayerProcessorOrganizer(); + this._transformer = new ObservationsTransformer(defaultArcGISPluginConfig, console); + this._geometryChangeHandler = new GeometryChangedHandler(this._transformer); + this._eventDeletionHandler = new EventDeletionHandler(this._console, defaultArcGISPluginConfig); + } + + /** + * Gets the current configuration from the database. + * @returns The current configuration from the database. + */ + public async safeGetConfig(): Promise { + const state = await this._stateRepo.get(); + if (!state) return await this._stateRepo.put(defaultArcGISPluginConfig); + return await this._stateRepo.get().then((state) => state ? state : this._stateRepo.put(defaultArcGISPluginConfig)); + } + + /** + * Puts a new confguration in the state repo. + * @param newConfig The new config to put into the state repo. + */ + public async putConfig(newConfig: ArcGISPluginConfig): Promise { + return await this._stateRepo.put(newConfig); + } + + /** + * Updates the confguration in the state repo. + * @param newConfig The new config to put into the state repo. + */ + public async patchConfig(newConfig: ArcGISPluginConfig): Promise { + return await this._stateRepo.patch(newConfig); + } + + /** + * Gets the current configuration and updates the processor if needed + * @returns The current configuration from the database. + */ + private async updateConfig(): Promise { + const config = await this.safeGetConfig() + + // Include form definitions while detecting changes in config + const eventForms = await this._eventRepo.findAll(); + const fullConfig = { ...config, eventForms }; + + const configJson = JSON.stringify(fullConfig) + if (this._previousConfig == null || this._previousConfig != configJson) { + this._transformer = new ObservationsTransformer(config, console); + this._geometryChangeHandler = new GeometryChangedHandler(this._transformer); + this._eventDeletionHandler = new EventDeletionHandler(this._console, config); + this._layerProcessors = []; + await this.getFeatureServiceLayers(config); + this._previousConfig = configJson + this._firstRun = true; + } + return config + } + + /** + * Starts the processor. + */ + async start() { + this._isRunning = true; + this._firstRun = true; + this.processAndScheduleNext(); + } + + /** + * Stops the processor. + */ + stop() { + this._isRunning = false; + clearTimeout(this._nextTimeout); + } + + /** + * Gets information on all the configured features service layers. + * @param config The plugins configuration. + */ + private async getFeatureServiceLayers(config: ArcGISPluginConfig) { + for (const service of config.featureServices) { + try { + const identityManager = await this._identityService.signin(service) + const response = await request(service.url, { authentication: identityManager }) + this.handleFeatureService(response, service, config) + } catch (err) { + console.error(err) + } + } + } + + /** + * Called when information on a feature service is returned from an arc server. + * @param featureService The feature service. + * @param featureServiceConfig The feature service config. + * @param config The plugin configuration. + */ + private async handleFeatureService(featureService: FeatureServiceResult, featureServiceConfig: FeatureServiceConfig, config: ArcGISPluginConfig) { + + if (featureService.layers != null) { + + const serviceLayers = new Map() + const admin = new FeatureServiceAdmin(config, this._identityService, this._console) + + let maxId = -1 + for (const layer of featureService.layers) { + serviceLayers.set(layer.id, layer) + serviceLayers.set(layer.name, layer) + maxId = Math.max(maxId, layer.id) + } + + for (const featureLayer of featureServiceConfig.layers) { + const eventNames: string[] = [] + const events = featureLayer.events + if (events != null) { + for (const event of events) { + const eventId = Number(event); + if (isNaN(eventId)) { + eventNames.push(String(event)); + } else { + const mageEvent = await this._eventRepo.findById(eventId) + if (mageEvent != null) { + eventNames.push(mageEvent.name); + } + } + } + } + if (eventNames.length > 0) { + featureLayer.events = eventNames + } + + const layer = serviceLayers.get(featureLayer.layer) + + let layerId = undefined + if (layer != null) { + layerId = layer.id + } else { + layerId = await admin.createLayer(featureServiceConfig, featureLayer, maxId + 1, this._eventRepo) + maxId = Math.max(maxId, layerId) + } + + if (layerId != null) { + featureLayer.layer = layerId + const identityManager = await this._identityService.signin(featureServiceConfig) + const featureService = new FeatureService(console, featureServiceConfig, identityManager) + const layerInfo = await featureService.queryLayerInfo(layerId); + const url = `${featureServiceConfig.url}/${layerId}`; + this.handleLayerInfo(url, featureServiceConfig, featureLayer, layerInfo, config); + } + } + } + } + + /** + * Called when information on a feature layer is returned from an arc server. + * @param url The layer url. + * @param featureServiceConfig The feature service config. + * @param featureLayer The feature layer configuration. + * @param layerInfo The information on a layer. + * @param config The plugins configuration. + */ + private async handleLayerInfo(url: string, featureServiceConfig: FeatureServiceConfig, featureLayer: FeatureLayerConfig, layerInfo: LayerInfoResult, config: ArcGISPluginConfig) { + if (layerInfo.geometryType != null) { + const events = featureLayer.events as string[] + const admin = new FeatureServiceAdmin(config, this._identityService, this._console) + await admin.updateLayer(featureServiceConfig, featureLayer, layerInfo, this._eventRepo) + const info = new LayerInfo(url, events, layerInfo) + const identityManager = await this._identityService.signin(featureServiceConfig) + const layerProcessor = new FeatureLayerProcessor(info, config, identityManager, this._console); + this._layerProcessors.push(layerProcessor); + // clearTimeout(this._nextTimeout); // TODO why is this needed? + // this.scheduleNext(config); // TODO why is this needed when processAndScheduleNext is called upstream and ends with scheduleNext() This causes a query before updateLayer. + } + } + + /** + * Processes any new observations and then schedules its next run if it hasn't been stopped. + */ + private async processAndScheduleNext() { + const config = await this.updateConfig(); + if (this._isRunning) { + if (config.enabled && this._layerProcessors.length > 0) { + this._console.info('ArcGIS plugin checking for any pending updates or adds'); + for (const layerProcessor of this._layerProcessors) { + layerProcessor.processPendingUpdates(); + } + this._console.info('ArcGIS plugin processing new observations...'); + const activeEvents = await this._eventRepo.findActiveEvents(); + this._eventDeletionHandler.checkForEventDeletion(activeEvents, this._layerProcessors, this._firstRun); + const eventsToProcessors = this._organizer.organize(activeEvents, this._layerProcessors); + const nextQueryTime = Date.now(); + for (const pair of eventsToProcessors) { + this._console.info('ArcGIS getting newest observations for event ' + pair.event.name); + const obsRepo = await this._obsRepos(pair.event.id); + const pagingSettings = { + pageSize: config.batchSize, + pageIndex: 0, + includeTotalCount: true + } + let morePages = true; + let numberLeft = 0; + while (morePages) { + numberLeft = await this.queryAndSend(config, pair.featureLayerProcessors, obsRepo, pagingSettings, numberLeft); + morePages = numberLeft > 0; + } + } + + for (const layerProcessor of this._layerProcessors) { + layerProcessor.lastTimeStamp = nextQueryTime; + } + + this._firstRun = false; + + // ArcGISIndentityManager access tokens may have been updated check and save + this._identityService.updateIndentityManagers() + } + this.scheduleNext(config); + } + } + + private scheduleNext(config: ArcGISPluginConfig) { + if (this._isRunning) { + let interval = config.intervalSeconds; + if (this._firstRun && config.featureServices.length > 0) { + interval = config.startupIntervalSeconds; + } else { + for (const layerProcessor of this._layerProcessors) { + if (layerProcessor.hasPendingUpdates()) { + interval = config.updateIntervalSeconds; + break; + } + } + } + this._nextTimeout = setTimeout(() => { this.processAndScheduleNext() }, interval * 1000); + } + } + + /** + * Queries for new observations and sends them to any configured arc servers. + * @param config The plugin configuration. + * @param layerProcessors The layer processors to use when processing arc objects. + * @param obsRepo The observation repo for an event. + * @param pagingSettings Current paging settings. + * @param numberLeft The number of observations left to query and send to arc. + * @returns The number of observations still needing to be queried and sent to arc. + */ + private async queryAndSend(config: ArcGISPluginConfig, layerProcessors: FeatureLayerProcessor[], obsRepo: EventScopedObservationRepository, pagingSettings: PagingParameters, numberLeft: number): Promise { + let newNumberLeft = numberLeft; + + let queryTime = -1; + for (const layerProcessor of layerProcessors) { + if (queryTime == -1 || layerProcessor.lastTimeStamp < queryTime) { + queryTime = layerProcessor.lastTimeStamp; + } + } + + let latestObs = await obsRepo.findLastModifiedAfter(queryTime, pagingSettings); + if (latestObs != null && latestObs.totalCount != null && latestObs.totalCount > 0) { + if (pagingSettings.pageIndex == 0) { + this._console.info('ArcGIS newest observation count ' + latestObs.totalCount); + newNumberLeft = latestObs.totalCount; + } + const observations = latestObs.items + const mageEvent = await this._eventRepo.findById(obsRepo.eventScope) + const eventTransform = new EventTransform(config, mageEvent) + const arcObjects = new ArcObjects() + this._geometryChangeHandler.checkForGeometryChange(observations, arcObjects, layerProcessors, this._firstRun); + for (let i = 0; i < observations.length; i++) { + const observation = observations[i] + let deletion = false + if (observation.states.length > 0) { + deletion = observation.states[0].name.startsWith('archive') + } + if (deletion) { + const arcObservation = this._transformer.createObservation(observation) + arcObjects.deletions.push(arcObservation) + } else { + let user = null + if (observation.userId != null) { + user = await this._userRepo.findById(observation.userId) + } + const arcObservation = this._transformer.transform(observation, eventTransform, user) + arcObjects.add(arcObservation) + } + } + arcObjects.firstRun = this._firstRun; + for (const layerProcessor of layerProcessors) { + layerProcessor.processArcObjects(JSON.parse(JSON.stringify(arcObjects))); + } + newNumberLeft -= latestObs.items.length; + pagingSettings.pageIndex++; + } else { + this._console.info('ArcGIS no new observations') + } + + return newNumberLeft; + } } \ No newline at end of file diff --git a/plugins/arcgis/service/src/ObservationsSender.ts b/plugins/arcgis/service/src/ObservationsSender.ts index 403ac484e..ae9ab4b15 100644 --- a/plugins/arcgis/service/src/ObservationsSender.ts +++ b/plugins/arcgis/service/src/ObservationsSender.ts @@ -1,7 +1,6 @@ import { ArcGISPluginConfig } from "./ArcGISPluginConfig"; import { ArcObjects } from './ArcObjects'; import { ArcObservation, ArcAttachment } from './ArcObservation'; -import { HttpClient } from './HttpClient'; import { LayerInfo } from "./LayerInfo"; import { EditResult } from './EditResult'; import { AttachmentInfosResult, AttachmentInfo } from './AttachmentInfosResult'; @@ -9,6 +8,8 @@ import environment from '@ngageoint/mage.service/lib/environment/env' import fs from 'fs' import path from 'path' import FormData from 'form-data'; +import { ArcGISIdentityManager, IFeature, request } from "@esri/arcgis-rest-request" +import { addFeatures, updateFeatures, deleteFeatures, getAttachments, updateAttachment, addAttachment, deleteAttachments } from "@esri/arcgis-rest-feature-service"; /** * Class that transforms observations into a json string that can then be sent to an arcgis server. @@ -35,11 +36,6 @@ export class ObservationsSender { */ private _console: Console; - /** - * Used to send the observations to an arc server. - */ - private _httpClient: HttpClient; - /** * The attachment base directory */ @@ -50,20 +46,22 @@ export class ObservationsSender { */ private _config: ArcGISPluginConfig; + private _identityManager: ArcGISIdentityManager; + /** * Constructor. * @param layerInfo The layer info. * @param config The plugins configuration. * @param console Used to log to the console. */ - constructor(layerInfo: LayerInfo, config: ArcGISPluginConfig, console: Console) { + constructor(layerInfo: LayerInfo, config: ArcGISPluginConfig, identityManager: ArcGISIdentityManager, console: Console) { this._url = layerInfo.url; this._urlAdd = this._url + '/addFeatures'; this._urlUpdate = this._url + '/updateFeatures'; this._console = console; - this._httpClient = new HttpClient(console, layerInfo.token); this._attachmentDirectory = environment.attachmentBaseDirectory; this._config = config; + this._identityManager = identityManager; } /** @@ -72,13 +70,14 @@ export class ObservationsSender { * @param observations The observations to convert. */ sendAdds(observations: ArcObjects) { - const contentString = 'gdbVersion=&rollbackOnFailure=true&timeReferenceUnknownClient=false&features=' + JSON.stringify(observations.objects); - this._console.info('ArcGIS addFeatures url ' + this._urlAdd); - this._console.info('ArcGIS addFeatures content ' + contentString); - let responseHandler = this.addResponseHandler(observations) - this._httpClient.sendPostHandleResponse(this._urlAdd, contentString, responseHandler); + let responseHandler = this.addResponseHandler(observations); + addFeatures({ + url: this._url, + authentication: this._identityManager, + features: observations.objects as IFeature[] + }).then(responseHandler).catch((error) => this._console.error(error)); } /** @@ -88,30 +87,29 @@ export class ObservationsSender { * @returns The json string of the observations. */ sendUpdates(observations: ArcObjects) { - const contentString = 'gdbVersion=&rollbackOnFailure=true&timeReferenceUnknownClient=false&features=' + JSON.stringify(observations.objects); - this._console.info('ArcGIS updateFeatures url ' + this._urlUpdate); - this._console.info('ArcGIS updateFeatures content ' + contentString); - let responseHandler = this.updateResponseHandler(observations) - this._httpClient.sendPostHandleResponse(this._urlUpdate, contentString, responseHandler); + let responseHandler = this.updateResponseHandler(observations); + updateFeatures({ + url: this._url, + authentication: this._identityManager, + features: observations.objects as IFeature[] + }).then(responseHandler).catch((error) => this._console.error(error)); } /** * Delete an observation. * @param id The observation id. */ - sendDelete(id: string) { - + sendDelete(id: number) { const url = this._url + '/deleteFeatures' - this._console.info('ArcGIS deleteFeatures url ' + url + ', ' + this._config.observationIdField + ': ' + id) - const form = new FormData() - form.append('where', this._config.observationIdField + ' LIKE\'' + id + "%\'") - - this._httpClient.sendPostForm(url, form) - + deleteFeatures({ + url, + authentication: this._identityManager, + objectIds: [id] + }).catch((error) => this._console.error(error)); } /** @@ -132,8 +130,7 @@ export class ObservationsSender { form.append('where', this._config.eventIdField + '=' + id) } - this._httpClient.sendPostForm(url, form) - + this.sendDelete(id); } /** @@ -163,8 +160,8 @@ export class ObservationsSender { private responseHandler(observations: ArcObjects, update: boolean): (chunk: any) => void { const console = this._console return (chunk: any) => { - console.log('ArcGIS ' + (update ? 'Update' : 'Add') + ' Response: ' + chunk) - const response = JSON.parse(chunk) + console.log('ArcGIS ' + (update ? 'Update' : 'Add') + ' Response: ' + JSON.stringify(chunk)) + const response = chunk const results = response[update ? 'updateResults' : 'addResults'] as EditResult[] if (results != null) { const obs = observations.observations @@ -209,15 +206,16 @@ export class ObservationsSender { * @param objectId The arc object id of the observation. */ private queryAndUpdateAttachments(observation: ArcObservation, objectId: number) { - // Query for existing attachments const queryUrl = this._url + '/' + objectId + '/attachments' - this._httpClient.sendGetHandleResponse(queryUrl, (chunk) => { - this._console.info('ArcGIS response for ' + queryUrl + ' ' + chunk) - const result = JSON.parse(chunk) as AttachmentInfosResult + getAttachments({ + url: this._url, + authentication: this._identityManager, + featureId: objectId + }).then((response) => { + const result = response as AttachmentInfosResult this.updateAttachments(observation, objectId, result.attachmentInfos) - }) - + }).catch((error) => this._console.error(error)); } /** @@ -270,8 +268,23 @@ export class ObservationsSender { * @param attachment The observation attachment. * @param objectId The arc object id of the observation. */ - private sendAttachment(attachment: ArcAttachment, objectId: number) { - this.sendAttachmentPost(attachment, objectId, 'addAttachment', new FormData()) + private async sendAttachment(attachment: ArcAttachment, objectId: number) { + if (attachment.contentLocator) { + const file = path.join(this._attachmentDirectory, attachment.contentLocator!) + + const fileName = this.attachmentFileName(attachment) + this._console.info('ArcGIS ' + request + ' file ' + fileName + ' from ' + file) + + const readStream = await fs.openAsBlob(file) + const attachmentFile = new File([readStream], fileName) + + addAttachment({ + url: this._url, + authentication: this._identityManager, + featureId: objectId, + attachment: attachmentFile + }).catch((error) => this._console.error(error)); + } } /** @@ -280,45 +293,24 @@ export class ObservationsSender { * @param objectId The arc object id of the observation. * @param attachmentId The observation arc attachment id. */ - private updateAttachment(attachment: ArcAttachment, objectId: number, attachmentId: number) { - - const form = new FormData() - form.append('attachmentId', attachmentId) - - this.sendAttachmentPost(attachment, objectId, 'updateAttachment', form) - - } - - /** - * Send an observation attachment post request. - * @param attachment The observation attachment. - * @param objectId The arc object id of the observation. - * @param request The attachment request type. - * @param form The request form data - */ - private sendAttachmentPost(attachment: ArcAttachment, objectId: number, request: string, form: FormData) { - - if (attachment.contentLocator != null) { - - const url = this._url + '/' + objectId + '/' + request - + private async updateAttachment(attachment: ArcAttachment, objectId: number, attachmentId: number) { + if (attachment.contentLocator) { const file = path.join(this._attachmentDirectory, attachment.contentLocator!) const fileName = this.attachmentFileName(attachment) - - this._console.info('ArcGIS ' + request + ' url ' + url) this._console.info('ArcGIS ' + request + ' file ' + fileName + ' from ' + file) - - const readStream = fs.createReadStream(file) - - form.append('attachment', readStream, { - filename: fileName - }) - - this._httpClient.sendPostForm(url, form) - + + const readStream = await fs.openAsBlob(file) + const attachmentFile = new File([readStream], fileName) + + updateAttachment({ + url: this._url, + authentication: this._identityManager, + featureId: objectId, + attachmentId, + attachment: attachmentFile + }).catch((error) => this._console.error(error)); } - } /** @@ -334,7 +326,7 @@ export class ObservationsSender { attachmentIds.push(attachmentInfo.id) } - this.deleteAttachmentIds(objectId, attachmentIds) + this.deleteAttachmentIds(objectId, attachmentIds); } /** @@ -343,24 +335,14 @@ export class ObservationsSender { * @param attachmentIds The arc attachment ids. */ private deleteAttachmentIds(objectId: number, attachmentIds: number[]) { - - const url = this._url + '/' + objectId + '/deleteAttachments' - - let ids = '' - for (const id of attachmentIds) { - if (ids.length > 0) { - ids += ', ' - } - ids += id - } - - this._console.info('ArcGIS deleteAttachments url ' + url + ', ids: ' + ids) - - const form = new FormData() - form.append('attachmentIds', ids) - - this._httpClient.sendPostForm(url, form) - + this._console.info('ArcGIS deleteAttachments ' + attachmentIds) + + deleteAttachments({ + url: this._url, + authentication: this._identityManager, + featureId: objectId, + attachmentIds + }).catch((error) => this._console.error(error)); } /** @@ -369,7 +351,6 @@ export class ObservationsSender { * @return attachment file name. */ private attachmentFileName(attachment: ArcAttachment): string { - let fileName = attachment.field + "_" + attachment.name const extensionIndex = attachment.contentLocator.lastIndexOf('.') diff --git a/plugins/arcgis/service/src/ObservationsTransformer.ts b/plugins/arcgis/service/src/ObservationsTransformer.ts index 49abf295b..529b5a0d4 100644 --- a/plugins/arcgis/service/src/ObservationsTransformer.ts +++ b/plugins/arcgis/service/src/ObservationsTransformer.ts @@ -328,7 +328,9 @@ export class ObservationsTransformer { if (fields != undefined) { const fieldTitle = fields.get(title) if (fieldTitle != undefined) { - title = fieldTitle + const sanitizedName = ObservationsTransformer.replaceSpaces(fieldTitle) + const sanitizedFormName = ObservationsTransformer.replaceSpaces(fields.name) + title = `${sanitizedFormName}_${sanitizedName}`.toLowerCase() } } if (field.type == FormFieldType.Geometry) { diff --git a/plugins/arcgis/service/src/index.ts b/plugins/arcgis/service/src/index.ts index c3d9e02f6..b70e32333 100644 --- a/plugins/arcgis/service/src/index.ts +++ b/plugins/arcgis/service/src/index.ts @@ -4,14 +4,12 @@ import { ObservationRepositoryToken } from '@ngageoint/mage.service/lib/plugins. import { MageEventRepositoryToken } from '@ngageoint/mage.service/lib/plugins.api/plugins.api.events' import { UserRepositoryToken } from '@ngageoint/mage.service/lib/plugins.api/plugins.api.users' import { SettingPermission } from '@ngageoint/mage.service/lib/entities/authorization/entities.permissions' -import { ArcGISPluginConfig } from './ArcGISPluginConfig' -import { AuthType } from './ArcGISConfig' import { ObservationProcessor } from './ObservationProcessor' import { ArcGISIdentityManager, request } from "@esri/arcgis-rest-request" -import { FeatureServiceConfig, OAuthAuthConfig } from './ArcGISConfig' +import { FeatureServiceConfig } from './ArcGISConfig' import { URL } from "node:url" import express from 'express' -import { getIdentityManager, getPortalUrl } from './ArcGISIdentityManagerFactory' +import { ArcGISIdentityService, createArcGISIdentityService, getPortalUrl } from './ArcGISService' const logPrefix = '[mage.arcgis]' const logMethods = ['log', 'debug', 'info', 'warn', 'error'] as const @@ -37,17 +35,15 @@ const InjectedServices = { const pluginWebRoute = "plugins/@ngageoint/mage.arcgis.service" -const sanitizeFeatureService = (config: FeatureServiceConfig, type: AuthType): FeatureServiceConfig => { - if (type === AuthType.OAuth) { - const newAuth = Object.assign({}, config.auth) as OAuthAuthConfig; - delete newAuth.refreshToken; - delete newAuth.refreshTokenExpires; - return { - ...config, - auth: newAuth - } - } - return config; +const sanitizeFeatureService = async (config: FeatureServiceConfig, identityService: ArcGISIdentityService): Promise> => { + let authenticated = false + try { + await identityService.signin(config) + authenticated = true + } catch(ignore) {} + + const { identityManager, ...sanitized } = config; + return { ...sanitized, authenticated } } /** @@ -65,11 +61,9 @@ const arcgisPluginHooks: InitPluginHook = { init: async (services): Promise => { console.info('Intializing ArcGIS plugin...') const { stateRepo, eventRepo, obsRepoForEvent, userRepo } = services - // TODO - // - Update layer token to get token from identity manager - // - Move plugins/arcgis/web-app/projects/main/src/lib/arc-layer/arc-layer.component.ts addLayer to helper file and use instead of encodeURIComponent - const processor = new ObservationProcessor(stateRepo, eventRepo, obsRepoForEvent, userRepo, console) + const identityService = createArcGISIdentityService(stateRepo) + const processor = new ObservationProcessor(stateRepo, eventRepo, obsRepoForEvent, userRepo, identityService, console) processor.start() return { webRoutes: { @@ -98,7 +92,6 @@ const arcgisPluginHooks: InitPluginHook = { routes.get('/oauth/authenticate', async (req, res) => { const code = req.query.code as string - // TODO is clientId here in req or response let state: { url: string, clientId: string } try { const { url, clientId } = JSON.parse(req.query.state as string) @@ -117,27 +110,24 @@ const arcgisPluginHooks: InitPluginHook = { ArcGISIdentityManager.exchangeAuthorizationCode(creds, code).then(async (idManager: ArcGISIdentityManager) => { let service = config.featureServices.find(service => service.url === state.url) if (!service) { - service = { url: state.url, layers: [] } - config.featureServices.push(service) + service = { + url: state.url, + identityManager: idManager.serialize(), + layers: [] + } + } else { + service.identityManager = idManager.serialize() } - service.auth = { - ...service.auth, - type: AuthType.OAuth, - clientId: state.clientId, - authToken: idManager.token, - authTokenExpires: idManager.tokenExpires.getTime(), - refreshToken: idManager.refreshToken, - refreshTokenExpires: idManager.refreshTokenExpires.getTime() - } + config.featureServices.push(service) await processor.putConfig(config) - // TODO: This seems like a bad idea to send the access tokens to the front end. It has no use for them and could potentially be a security concern + const sanitizedService = await sanitizeFeatureService(service, identityService) res.send(` @@ -163,45 +153,70 @@ const arcgisPluginHooks: InitPluginHook = { .get(async (req, res, next) => { console.info('Getting ArcGIS plugin config...') const config = await processor.safeGetConfig() - config.featureServices = config.featureServices.map((service) => sanitizeFeatureService(service, AuthType.OAuth)); - res.json(config) + const { featureServices, ...remaining } = config + + const sanitizeFeatureServices = await Promise.all( + featureServices.map(async (service) => await sanitizeFeatureService(service, identityService)) + ) + + res.json({ ...remaining, featureServices: sanitizeFeatureServices }) }) .put(async (req, res, next) => { console.info('Applying ArcGIS plugin config...') - const arcConfig = req.body as ArcGISPluginConfig - const configString = JSON.stringify(arcConfig) - processor.patchConfig(arcConfig) + const config = await stateRepo.get() + const { featureServices: updatedServices, ...updateConfig } = req.body + + // Map exisiting identityManager, client does not send this + const featureServices: FeatureServiceConfig[] = updatedServices.map((updateService: FeatureServiceConfig) => { + const existingService = config.featureServices.find((featureService: FeatureServiceConfig) => featureService.url === updateService.url) + return { + url: updateService.url, + layers: updateService.layers, + identityManager: existingService?.identityManager + } + }) + + await stateRepo.patch({ ...updateConfig, featureServices }) + + // Sync configuration with feature servers by restarting observation processor + processor.stop() + processor.start() + res.sendStatus(200) }) routes.post('/featureService/validate', async (req, res) => { const config = await processor.safeGetConfig() - const { url, auth = {} } = req.body - const { token, username, password } = auth + const { url, token, username, password } = req.body if (!URL.canParse(url)) { return res.send('Invalid feature service url').status(400) } let service: FeatureServiceConfig + let identityManager: ArcGISIdentityManager if (token) { - service = { url, layers: [], auth: { type: AuthType.Token, token } } + identityManager = await ArcGISIdentityManager.fromToken({ token }) + service = { url, layers: [], identityManager: identityManager.serialize() } } else if (username && password) { - service = { url, layers: [], auth: { type: AuthType.UsernamePassword, username, password } } + identityManager = await ArcGISIdentityManager.signIn({ + username, + password, + portal: getPortalUrl(url) + }) + service = { url, layers: [], identityManager: identityManager.serialize() } } else { return res.sendStatus(400) } try { - // Create the IdentityManager instance to validate credentials - await getIdentityManager(service!, processor) let existingService = config.featureServices.find(service => service.url === url) - if (existingService) { - existingService = { ...existingService } - } else { + if (!existingService) { config.featureServices.push(service) } + await processor.patchConfig(config) - return res.send(sanitizeFeatureService(service, AuthType.OAuth)) + const sanitized = await sanitizeFeatureService(service, identityService) + return res.send(sanitized) } catch (err) { return res.send('Invalid credentials provided to communicate with feature service').status(400) } @@ -216,7 +231,7 @@ const arcgisPluginHooks: InitPluginHook = { } try { - const identityManager = await getIdentityManager(featureService, processor) + const identityManager = await identityService.signin(featureService) const response = await request(url, { authentication: identityManager }) diff --git a/plugins/arcgis/web-app/projects/main/src/lib/ArcGISConfig.ts b/plugins/arcgis/web-app/projects/main/src/lib/ArcGISConfig.ts index c59cb6b91..3c2ed0f9b 100644 --- a/plugins/arcgis/web-app/projects/main/src/lib/ArcGISConfig.ts +++ b/plugins/arcgis/web-app/projects/main/src/lib/ArcGISConfig.ts @@ -9,30 +9,14 @@ export interface FeatureServiceConfig { url: string /** - * Username and password for ArcGIS authentication + * Flag indicating valid authentication */ - auth?: ArcGISAuthConfig - - /** - * Create layers that don't exist - */ - createLayers?: boolean - - /** - * The administration url to the arc feature service. - */ - adminUrl?: string - - /** - * Administration access token - */ - adminToken?: string + authenticated: boolean /** * The feature layers. */ layers: FeatureLayerConfig[] - } /** @@ -50,92 +34,12 @@ export interface FeatureLayerConfig { */ geometryType?: string - /** - * Access token - */ - token?: string // TODO - can this be removed? Will Layers have a token too? /** * The event ids or names that sync to this arc feature layer. */ events?: (number|string)[] - - /** - * Add layer fields from form fields - */ - addFields?: boolean - - /** - * Delete editable layer fields missing from form fields - */ - deleteFields?: boolean - } -export enum AuthType { - Token = 'token', - UsernamePassword = 'usernamePassword', - OAuth = 'oauth' - } - - -/** - * Contains token-based authentication configuration. - */ -export interface TokenAuthConfig { - type: AuthType.Token - token: string - authTokenExpires?: string -} - -/** - * Contains username and password for ArcGIS server authentication. - */ -export interface UsernamePasswordAuthConfig { - type: AuthType.UsernamePassword - /** - * The username for authentication. - */ - username: string - - /** - * The password for authentication. - */ - password: string -} - -/** - * Contains OAuth authentication configuration. - */ -export interface OAuthAuthConfig { - type: AuthType.OAuth - /** - * The Client Id for OAuth - */ - clientId: string - /** - * The redirectUri for OAuth - */ - redirectUri?: string - - /** - * The temporary auth token for OAuth - */ - authToken?: string - - /** - * The expiration date for the temporary token - */ - authTokenExpires?: string -} - -/** - * Union type for authentication configurations. - */ -export type ArcGISAuthConfig = - | TokenAuthConfig - | UsernamePasswordAuthConfig - | OAuthAuthConfig - /** * Attribute configurations */ diff --git a/plugins/arcgis/web-app/projects/main/src/lib/arc-layer/arc-layer-dialog.component.html b/plugins/arcgis/web-app/projects/main/src/lib/arc-layer/arc-layer-dialog.component.html index 5644e38b9..23fd57692 100644 --- a/plugins/arcgis/web-app/projects/main/src/lib/arc-layer/arc-layer-dialog.component.html +++ b/plugins/arcgis/web-app/projects/main/src/lib/arc-layer/arc-layer-dialog.component.html @@ -1,27 +1,37 @@
-
ArcGIS Feature Service
+
+ +
+
ArcGIS Feature Service
- + - Feature Service + Feature Service + {{featureService?.url}} + +
+ error_outline Credentials invalid or expired +
+
URL - + URL is required Authentication - - {{authenticationType.title}} + + {{authenticationState.text}} @@ -29,9 +39,9 @@
- Token + API Key - Token is required + API Key is required
@@ -61,7 +71,8 @@
@@ -85,7 +96,7 @@
diff --git a/plugins/arcgis/web-app/projects/main/src/lib/arc-layer/arc-layer-dialog.component.scss b/plugins/arcgis/web-app/projects/main/src/lib/arc-layer/arc-layer-dialog.component.scss index a0671015d..c6e13a0ea 100644 --- a/plugins/arcgis/web-app/projects/main/src/lib/arc-layer/arc-layer-dialog.component.scss +++ b/plugins/arcgis/web-app/projects/main/src/lib/arc-layer/arc-layer-dialog.component.scss @@ -6,6 +6,12 @@ mat-dialog-content { flex: 1 } +.cancel { + position: absolute; + top: -16px; + right: -16px; +} + .dialog { min-width: 700px; min-height: 450px; @@ -24,6 +30,13 @@ mat-dialog-content { margin-bottom: 16px; } +.invalid-credentials { + color: #F44336; + display: flex; + align-items: center; + gap: 8px; +} + .actions { width: 100%; display: flex; diff --git a/plugins/arcgis/web-app/projects/main/src/lib/arc-layer/arc-layer-dialog.component.ts b/plugins/arcgis/web-app/projects/main/src/lib/arc-layer/arc-layer-dialog.component.ts index 8d4428a9e..8a71ba23e 100644 --- a/plugins/arcgis/web-app/projects/main/src/lib/arc-layer/arc-layer-dialog.component.ts +++ b/plugins/arcgis/web-app/projects/main/src/lib/arc-layer/arc-layer-dialog.component.ts @@ -2,14 +2,20 @@ import { Component, Inject, ViewChild } from '@angular/core'; import { FormControl, FormGroup, Validators } from '@angular/forms'; import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog'; import { MatSelectionList } from '@angular/material/list'; -import { AuthType, FeatureServiceConfig } from '../ArcGISConfig'; +import { FeatureServiceConfig } from '../ArcGISConfig'; import { ArcService, FeatureLayer } from '../arc.service'; enum State { Validate, Layers } -interface AuthenticationType { - title: string - value: string +enum AuthenticationType { + Token = 'token', + UsernamePassword = 'usernamePassword', + OAuth = 'oauth' +} + +type AuthenticationState = { + text: string + value: AuthenticationType } export interface DialogData { @@ -27,16 +33,16 @@ export class ArcLayerDialogComponent { loading = false - AuthenticationType = AuthType - authenticationTypes: AuthenticationType[] = [{ - title: 'OAuth', - value: AuthType.OAuth + AuthenticationType = AuthenticationType + authenticationStates: AuthenticationState[] = [{ + text: 'OAuth', + value: AuthenticationType.OAuth },{ - title: 'Username/Password', - value: AuthType.UsernamePassword + text: 'Username/Password', + value: AuthenticationType.UsernamePassword },{ - title: 'Token', - value: AuthType.Token + text: 'API Key', + value: AuthenticationType.Token }] layerForm: FormGroup @@ -53,23 +59,20 @@ export class ArcLayerDialogComponent { if (data.featureService) { this.featureService = data.featureService } - const auth: any = this.featureService?.auth || {} - const { type, token, username, password, clientId } = auth - this.state = this.featureService === undefined ? State.Validate : State.Layers - // TODO update all fields with info from pass in service + this.state = this.featureService === undefined || !this.featureService.authenticated ? State.Validate : State.Layers this.layerForm = new FormGroup({ - url: new FormControl(this.featureService?.url, [Validators.required]), - authenticationType: new FormControl(type || AuthType.OAuth, [Validators.required]), + url: new FormControl({value: this.featureService?.url, disabled: this.featureService !== undefined }, [Validators.required]), + authenticationType: new FormControl('', [Validators.required]), token: new FormGroup({ - token: new FormControl(token, [Validators.required]) + token: new FormControl('', [Validators.required]) }), oauth: new FormGroup({ - clientId: new FormControl(clientId, [Validators.required]) + clientId: new FormControl('', [Validators.required]) }), local: new FormGroup({ - username: new FormControl(username, [Validators.required]), - password: new FormControl(password, [Validators.required]) + username: new FormControl('', [Validators.required]), + password: new FormControl('', [Validators.required]) }) }) @@ -99,25 +102,19 @@ export class ArcLayerDialogComponent { const { url, authenticationType } = this.layerForm.value switch (authenticationType) { - case AuthType.Token: { + case AuthenticationType.Token: { const { token } = this.layerForm.controls.token.value - this.featureService = { url, auth: { type: AuthType.Token, token }, layers: [] } - this.arcService.validateFeatureService(this.featureService).subscribe((service) => this.validated(service)) - + this.arcService.validateFeatureService({ url, token }).subscribe((service) => this.validated(service)) break; } - case AuthType.OAuth: { + case AuthenticationType.OAuth: { const { clientId } = this.layerForm.controls.oauth.value - this.featureService = { url, auth: { type: AuthType.OAuth, clientId }, layers: [] } this.arcService.oauth(url, clientId).subscribe((service) => this.validated(service)) - break; } - case AuthType.UsernamePassword: { + case AuthenticationType.UsernamePassword: { const { username, password } = this.layerForm.controls.local.value - this.featureService = { url, auth: { type: AuthType.UsernamePassword, username, password }, layers: [] } - this.arcService.validateFeatureService(this.featureService).subscribe((service) => this.validated(service)) - + this.arcService.validateFeatureService({url, username, password}).subscribe((service) => this.validated(service)) break; } } @@ -135,4 +132,8 @@ export class ArcLayerDialogComponent { }) this.dialogRef.close(this.featureService) } + + onCancel(): void { + this.dialogRef.close() + } } diff --git a/plugins/arcgis/web-app/projects/main/src/lib/arc-layer/arc-layer.component.html b/plugins/arcgis/web-app/projects/main/src/lib/arc-layer/arc-layer.component.html index 19f2ad22e..813b61fe6 100644 --- a/plugins/arcgis/web-app/projects/main/src/lib/arc-layer/arc-layer.component.html +++ b/plugins/arcgis/web-app/projects/main/src/lib/arc-layer/arc-layer.component.html @@ -1,39 +1,49 @@
-
-
-

Feature Layers

-

Add ArcGIS feature service urls and layers to sychronize MAGE data to.

-
-
- -
-
-
- There are no ArcGIS feature services currently being synchronized. -
-
+ Feature Layers + ArcGIS feature service URLs and layers to sychronize MAGE data. -
-
-
- {{service.url}} -
-
- -
-
- -
-
-
-
- {{layer.layer}} + +
+
+ place + arrow_right_alt + map
+ +
Create ArcGIS feature services used to synchronize MAGE data. Feature services you create will show up here.
-
-
+ + + + map +

{{service.url}}

+

+ + {{layer.layer}},  + +

+

+ Credentials invalid or expired +

+ + + open_in_new + + + + + + + + + + + +
\ No newline at end of file diff --git a/plugins/arcgis/web-app/projects/main/src/lib/arc-layer/arc-layer.component.scss b/plugins/arcgis/web-app/projects/main/src/lib/arc-layer/arc-layer.component.scss index b4b3333cb..3639a2efd 100644 --- a/plugins/arcgis/web-app/projects/main/src/lib/arc-layer/arc-layer.component.scss +++ b/plugins/arcgis/web-app/projects/main/src/lib/arc-layer/arc-layer.component.scss @@ -1,90 +1,50 @@ -.arc-layer-form { - display: flex; - flex-direction: column; - min-width: 950px; -} - -section { - >* { - margin-inline-start: 1em; - } - - header { - margin-inline-start: 0; - - h1, - h2, - h3, - h4, - h5, - h6 { - margin-block-end: 0; - } - - .subheading { - margin-block-start: 0; - opacity: 0.5; - } - - margin-block-end: 2em; - } - - margin-block-start: 2em; - margin-block-end: 2em; -} - -.arcLayer { - margin-block-start: 1em; - display: grid; - grid-template-columns: 8fr 1fr 1fr; - max-width: 600px; - margin-bottom: 20px; -} - -.arcLayer__value { - grid-column-start: 1; +mat-card { + margin-bottom: 16px; } -.arcLayer__edit { - grid-column-start: 2; - text-align: end; +.mat-list-icon { + color: rgba(0, 0, 0, 0.54); } -.arcLayer__action { - grid-column-start: 3; - text-align: end; +.no-mappings { + height: 300px; + width: 100%; + display: flex; + align-items: center; + justify-content: center; + flex-direction: column; } -.featureLayers { - display: grid; - opacity: 0.5; - font-size: 0.9em; +.no-mappings__icons { + display: flex; + direction: row; + margin-top: 16px; + margin-bottom: 48px; } -.edit-icon { - vertical-align: middle; +.no-mappings__icon { + font-size: 80px; + width: 80px; + height: 80px; + opacity: .6; } -.edit-button { - opacity: 0.5; - font-size: inherit; - height: inherit; - line-height: inherit; - width: inherit; +.no-mappings__icon--margin { + margin: 0 8px; } -.delete-icon { - vertical-align: middle; +.no-mappings__title { + font: 400 28px/36px Roboto, "Helvetica Neue", sans-serif; + width: 600px; + opacity: .6; + text-align: center; + margin: 0 16px 16px 16px; } -.delete-button { - opacity: 0.5; - font-size: inherit; - height: inherit; - line-height: inherit; - width: inherit; +.subtitle { + color: rgba(0, 0, 0, 0.54); } -mat-card { - margin-bottom: 16px; - } \ No newline at end of file +.invalid-credentials { + color: #F44336; +} \ No newline at end of file diff --git a/plugins/arcgis/web-app/projects/main/src/lib/arc-layer/arc-layer.component.ts b/plugins/arcgis/web-app/projects/main/src/lib/arc-layer/arc-layer.component.ts index 650e34b54..81da8d01e 100644 --- a/plugins/arcgis/web-app/projects/main/src/lib/arc-layer/arc-layer.component.ts +++ b/plugins/arcgis/web-app/projects/main/src/lib/arc-layer/arc-layer.component.ts @@ -31,10 +31,9 @@ export class ArcLayerComponent { onAddService() { this.dialog.open(ArcLayerDialogComponent, { - data: { - featureService: undefined - }, - autoFocus: false + data: { featureService: undefined }, + autoFocus: false, + disableClose: true }).afterClosed().subscribe(featureService => { if (featureService) { this.addFeatureService(featureService) @@ -42,12 +41,15 @@ export class ArcLayerComponent { }) } + onOpenService(featureService: FeatureLayerConfig) { + + } + onEditService(featureService: FeatureServiceConfig) { this.dialog.open(ArcLayerDialogComponent, { - data: { - featureService: featureService - }, - autoFocus: false + data: { featureService }, + autoFocus: false, + disableClose: true }).afterClosed().subscribe(featureService => { if (featureService) { this.addFeatureService(featureService) diff --git a/plugins/arcgis/web-app/projects/main/src/lib/arc.service.ts b/plugins/arcgis/web-app/projects/main/src/lib/arc.service.ts index bd2b831d6..cff0943a3 100644 --- a/plugins/arcgis/web-app/projects/main/src/lib/arc.service.ts +++ b/plugins/arcgis/web-app/projects/main/src/lib/arc.service.ts @@ -13,6 +13,7 @@ export interface ArcServiceInterface { fetchEvents(): Observable fetchPopulatedEvents(): Observable fetchFeatureServiceLayers(featureServiceUrl: string): Observable + validateFeatureService(request: ValidationRequest): Observable } export class MageEvent { @@ -37,6 +38,10 @@ export interface FeatureLayer { geometryType: string } +export type ValidationRequest = { + url: string +} & { token: string } | { username: string, password: string } + @Injectable({ providedIn: 'root' }) @@ -79,8 +84,8 @@ export class ArcService implements ArcServiceInterface { return subject.asObservable() } - validateFeatureService(service: FeatureServiceConfig): Observable { - return this.http.post(`${baseUrl}/featureService/validate`, service) + validateFeatureService(request: ValidationRequest): Observable { + return this.http.post(`${baseUrl}/featureService/validate`, request) } fetchEvents(): Observable { diff --git a/plugins/arcgis/web-app/projects/showcase/src/app/arc.service.mock.ts b/plugins/arcgis/web-app/projects/showcase/src/app/arc.service.mock.ts index 735b26dce..f4b582b96 100644 --- a/plugins/arcgis/web-app/projects/showcase/src/app/arc.service.mock.ts +++ b/plugins/arcgis/web-app/projects/showcase/src/app/arc.service.mock.ts @@ -1,6 +1,6 @@ import { Observable, of } from "rxjs"; import { Injectable } from '@angular/core' -import { ArcServiceInterface, FeatureLayer } from "../../../main/src/lib/arc.service"; +import { ArcServiceInterface, FeatureLayer, ValidationRequest } from "../../../main/src/lib/arc.service"; import { ArcGISPluginConfig, defaultArcGISPluginConfig } from '../../../main/src/lib/ArcGISPluginConfig'; import { MageEvent } from "../../../main/src/lib/arc.service"; @@ -152,4 +152,14 @@ export class MockArcService implements ArcServiceInterface { removeOperation(operationId: string): Observable { return of(defaultArcGISPluginConfig); } + + validateFeatureService(request: ValidationRequest): Observable { + return of({ + url: 'https://arcgis.mock.com/1', + authenticated: true, + layers: [{ + layer: 'Mock ArcGIS Layer 1' + }] + }) + } } \ No newline at end of file diff --git a/plugins/arcgis/web-app/projects/showcase/src/index.html b/plugins/arcgis/web-app/projects/showcase/src/index.html index 7c84f1b95..d466c3d05 100644 --- a/plugins/arcgis/web-app/projects/showcase/src/index.html +++ b/plugins/arcgis/web-app/projects/showcase/src/index.html @@ -8,7 +8,7 @@ - +