From 39a4c6f7af2ff26dd32b83b9630e097279a892f9 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Thu, 24 Oct 2024 15:08:06 -0700 Subject: [PATCH 01/41] [TM-1249] Scaffolding for the research service. --- .env.local.sample | 2 ++ .github/workflows/deploy-service.yml | 3 +- README.md | 1 + apps/api-gateway/lib/api-gateway-stack.ts | 4 +++ apps/research-service-e2e/.eslintrc.json | 18 +++++++++++ apps/research-service-e2e/jest.config.ts | 19 +++++++++++ apps/research-service-e2e/project.json | 17 ++++++++++ .../research-service/research-service.spec.ts | 10 ++++++ .../src/support/global-setup.ts | 10 ++++++ .../src/support/global-teardown.ts | 7 ++++ .../src/support/test-setup.ts | 10 ++++++ apps/research-service-e2e/tsconfig.json | 13 ++++++++ apps/research-service-e2e/tsconfig.spec.json | 9 ++++++ apps/research-service/.eslintrc.json | 18 +++++++++++ apps/research-service/Dockerfile | 15 +++++++++ apps/research-service/jest.config.ts | 11 +++++++ apps/research-service/project.json | 26 +++++++++++++++ apps/research-service/src/app.module.ts | 11 +++++++ apps/research-service/src/assets/.gitkeep | 0 .../src/health/health.controller.ts | 32 +++++++++++++++++++ .../src/health/health.module.ts | 9 ++++++ apps/research-service/src/main.ts | 31 ++++++++++++++++++ apps/research-service/tsconfig.app.json | 12 +++++++ apps/research-service/tsconfig.json | 16 ++++++++++ apps/research-service/tsconfig.spec.json | 14 ++++++++ apps/research-service/webpack.config.js | 20 ++++++++++++ nx.json | 3 +- 27 files changed, 339 insertions(+), 2 deletions(-) create mode 100644 apps/research-service-e2e/.eslintrc.json create mode 100644 apps/research-service-e2e/jest.config.ts create mode 100644 apps/research-service-e2e/project.json create mode 100644 apps/research-service-e2e/src/research-service/research-service.spec.ts create mode 100644 apps/research-service-e2e/src/support/global-setup.ts create mode 100644 apps/research-service-e2e/src/support/global-teardown.ts create mode 100644 apps/research-service-e2e/src/support/test-setup.ts create mode 100644 apps/research-service-e2e/tsconfig.json create mode 100644 apps/research-service-e2e/tsconfig.spec.json create mode 100644 apps/research-service/.eslintrc.json create mode 100644 apps/research-service/Dockerfile create mode 100644 apps/research-service/jest.config.ts create mode 100644 apps/research-service/project.json create mode 100644 apps/research-service/src/app.module.ts create mode 100644 apps/research-service/src/assets/.gitkeep create mode 100644 apps/research-service/src/health/health.controller.ts create mode 100644 apps/research-service/src/health/health.module.ts create mode 100644 apps/research-service/src/main.ts create mode 100644 apps/research-service/tsconfig.app.json create mode 100644 apps/research-service/tsconfig.json create mode 100644 apps/research-service/tsconfig.spec.json create mode 100644 apps/research-service/webpack.config.js diff --git a/.env.local.sample b/.env.local.sample index d90b147..135cd8d 100644 --- a/.env.local.sample +++ b/.env.local.sample @@ -3,6 +3,8 @@ USER_SERVICE_PROXY_PORT=4010 USER_SERVICE_PROXY_TARGET=http://host.docker.internal:$USER_SERVICE_PROXY_PORT JOB_SERVICE_PROXY_PORT=4020 JOB_SERVICE_PROXY_TARGET=http://host.docker.internal:$JOB_SERVICE_PROXY_PORT +RESEARCH_SERVICE_PROXY_PORT=4030 +RESEARCH_SERVICE_PROXY_TARGET=http://host.docker.internal:$RESEARCH_SERVICE_PROXY_PORT DB_HOST=localhost DB_PORT=3360 diff --git a/.github/workflows/deploy-service.yml b/.github/workflows/deploy-service.yml index 9d50059..b162933 100644 --- a/.github/workflows/deploy-service.yml +++ b/.github/workflows/deploy-service.yml @@ -9,8 +9,9 @@ on: type: choice required: true options: - - user-service - job-service + - research-service + - user-service env: description: 'Deployment target environment' type: choice diff --git a/README.md b/README.md index a5a585f..5ddccd5 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,7 @@ and main branches. * Make sure to kill your NX `serve` process and run `nx build api-gateway` before restarting it. * For deployment to AWS: * Add a Dockerfile in the new app directory. A simple copy and modify from user-service is sufficient + * Add the new service name to the "service" workflow input options in `deploy-service.yml` * In AWS: * Add ECR repositories for each env (follow the naming scheme from user-service, e.g. `terramatch-microservices/foo-service-staging`, etc) * Set the repo to Immutable diff --git a/apps/api-gateway/lib/api-gateway-stack.ts b/apps/api-gateway/lib/api-gateway-stack.ts index 937dc9d..2a2d8bc 100644 --- a/apps/api-gateway/lib/api-gateway-stack.ts +++ b/apps/api-gateway/lib/api-gateway-stack.ts @@ -35,6 +35,10 @@ const V3_SERVICES = { 'job-service': { targetHost: process.env.JOB_SERVICE_PROXY_TARGET ?? '', namespaces: ['jobs'] + }, + 'research-service': { + targetHost: process.env.RESEARCH_SERVICE_PROXY_TARGET ?? '', + namespaces: ['research'] } } diff --git a/apps/research-service-e2e/.eslintrc.json b/apps/research-service-e2e/.eslintrc.json new file mode 100644 index 0000000..9d9c0db --- /dev/null +++ b/apps/research-service-e2e/.eslintrc.json @@ -0,0 +1,18 @@ +{ + "extends": ["../../.eslintrc.json"], + "ignorePatterns": ["!**/*"], + "overrides": [ + { + "files": ["*.ts", "*.tsx", "*.js", "*.jsx"], + "rules": {} + }, + { + "files": ["*.ts", "*.tsx"], + "rules": {} + }, + { + "files": ["*.js", "*.jsx"], + "rules": {} + } + ] +} diff --git a/apps/research-service-e2e/jest.config.ts b/apps/research-service-e2e/jest.config.ts new file mode 100644 index 0000000..00ac442 --- /dev/null +++ b/apps/research-service-e2e/jest.config.ts @@ -0,0 +1,19 @@ +/* eslint-disable */ +export default { + displayName: 'research-service-e2e', + preset: '../../jest.preset.js', + globalSetup: '/src/support/global-setup.ts', + globalTeardown: '/src/support/global-teardown.ts', + setupFiles: ['/src/support/test-setup.ts'], + testEnvironment: 'node', + transform: { + '^.+\\.[tj]s$': [ + 'ts-jest', + { + tsconfig: '/tsconfig.spec.json', + }, + ], + }, + moduleFileExtensions: ['ts', 'js', 'html'], + coverageDirectory: '../../coverage/research-service-e2e', +}; diff --git a/apps/research-service-e2e/project.json b/apps/research-service-e2e/project.json new file mode 100644 index 0000000..c7e9489 --- /dev/null +++ b/apps/research-service-e2e/project.json @@ -0,0 +1,17 @@ +{ + "name": "research-service-e2e", + "$schema": "../../node_modules/nx/schemas/project-schema.json", + "projectType": "application", + "implicitDependencies": ["research-service"], + "targets": { + "e2e": { + "executor": "@nx/jest:jest", + "outputs": ["{workspaceRoot}/coverage/{e2eProjectRoot}"], + "options": { + "jestConfig": "apps/research-service-e2e/jest.config.ts", + "passWithNoTests": true + }, + "dependsOn": ["research-service:build"] + } + } +} diff --git a/apps/research-service-e2e/src/research-service/research-service.spec.ts b/apps/research-service-e2e/src/research-service/research-service.spec.ts new file mode 100644 index 0000000..e8ac2a6 --- /dev/null +++ b/apps/research-service-e2e/src/research-service/research-service.spec.ts @@ -0,0 +1,10 @@ +import axios from 'axios'; + +describe('GET /api', () => { + it('should return a message', async () => { + const res = await axios.get(`/api`); + + expect(res.status).toBe(200); + expect(res.data).toEqual({ message: 'Hello API' }); + }); +}); diff --git a/apps/research-service-e2e/src/support/global-setup.ts b/apps/research-service-e2e/src/support/global-setup.ts new file mode 100644 index 0000000..c1f5144 --- /dev/null +++ b/apps/research-service-e2e/src/support/global-setup.ts @@ -0,0 +1,10 @@ +/* eslint-disable */ +var __TEARDOWN_MESSAGE__: string; + +module.exports = async function () { + // Start services that that the app needs to run (e.g. database, docker-compose, etc.). + console.log('\nSetting up...\n'); + + // Hint: Use `globalThis` to pass variables to global teardown. + globalThis.__TEARDOWN_MESSAGE__ = '\nTearing down...\n'; +}; diff --git a/apps/research-service-e2e/src/support/global-teardown.ts b/apps/research-service-e2e/src/support/global-teardown.ts new file mode 100644 index 0000000..32ea345 --- /dev/null +++ b/apps/research-service-e2e/src/support/global-teardown.ts @@ -0,0 +1,7 @@ +/* eslint-disable */ + +module.exports = async function () { + // Put clean up logic here (e.g. stopping services, docker-compose, etc.). + // Hint: `globalThis` is shared between setup and teardown. + console.log(globalThis.__TEARDOWN_MESSAGE__); +}; diff --git a/apps/research-service-e2e/src/support/test-setup.ts b/apps/research-service-e2e/src/support/test-setup.ts new file mode 100644 index 0000000..07f2870 --- /dev/null +++ b/apps/research-service-e2e/src/support/test-setup.ts @@ -0,0 +1,10 @@ +/* eslint-disable */ + +import axios from 'axios'; + +module.exports = async function () { + // Configure axios for tests to use. + const host = process.env.HOST ?? 'localhost'; + const port = process.env.PORT ?? '3000'; + axios.defaults.baseURL = `http://${host}:${port}`; +}; diff --git a/apps/research-service-e2e/tsconfig.json b/apps/research-service-e2e/tsconfig.json new file mode 100644 index 0000000..ed633e1 --- /dev/null +++ b/apps/research-service-e2e/tsconfig.json @@ -0,0 +1,13 @@ +{ + "extends": "../../tsconfig.base.json", + "files": [], + "include": [], + "references": [ + { + "path": "./tsconfig.spec.json" + } + ], + "compilerOptions": { + "esModuleInterop": true + } +} diff --git a/apps/research-service-e2e/tsconfig.spec.json b/apps/research-service-e2e/tsconfig.spec.json new file mode 100644 index 0000000..d7f9cf2 --- /dev/null +++ b/apps/research-service-e2e/tsconfig.spec.json @@ -0,0 +1,9 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "../../dist/out-tsc", + "module": "commonjs", + "types": ["jest", "node"] + }, + "include": ["jest.config.ts", "src/**/*.ts"] +} diff --git a/apps/research-service/.eslintrc.json b/apps/research-service/.eslintrc.json new file mode 100644 index 0000000..9d9c0db --- /dev/null +++ b/apps/research-service/.eslintrc.json @@ -0,0 +1,18 @@ +{ + "extends": ["../../.eslintrc.json"], + "ignorePatterns": ["!**/*"], + "overrides": [ + { + "files": ["*.ts", "*.tsx", "*.js", "*.jsx"], + "rules": {} + }, + { + "files": ["*.ts", "*.tsx"], + "rules": {} + }, + { + "files": ["*.js", "*.jsx"], + "rules": {} + } + ] +} diff --git a/apps/research-service/Dockerfile b/apps/research-service/Dockerfile new file mode 100644 index 0000000..721783b --- /dev/null +++ b/apps/research-service/Dockerfile @@ -0,0 +1,15 @@ +FROM terramatch-microservices-base:nx-base AS builder + +ARG BUILD_FLAG +WORKDIR /app/builder +COPY . . +RUN npx nx build research-service ${BUILD_FLAG} + +FROM terramatch-microservices-base:nx-base + +ARG NODE_ENV +WORKDIR /app +COPY --from=builder /app/builder ./ +ENV NODE_ENV=${NODE_ENV} + +CMD ["node", "./dist/apps/research-service/main.js"] diff --git a/apps/research-service/jest.config.ts b/apps/research-service/jest.config.ts new file mode 100644 index 0000000..a670078 --- /dev/null +++ b/apps/research-service/jest.config.ts @@ -0,0 +1,11 @@ +/* eslint-disable */ +export default { + displayName: 'research-service', + preset: '../../jest.preset.js', + testEnvironment: 'node', + transform: { + '^.+\\.[tj]s$': ['ts-jest', { tsconfig: '/tsconfig.spec.json' }], + }, + moduleFileExtensions: ['ts', 'js', 'html'], + coverageDirectory: '../../coverage/apps/research-service', +}; diff --git a/apps/research-service/project.json b/apps/research-service/project.json new file mode 100644 index 0000000..8e97e47 --- /dev/null +++ b/apps/research-service/project.json @@ -0,0 +1,26 @@ +{ + "name": "research-service", + "$schema": "../../node_modules/nx/schemas/project-schema.json", + "sourceRoot": "apps/research-service/src", + "projectType": "application", + "tags": [], + "targets": { + "serve": { + "executor": "@nx/js:node", + "defaultConfiguration": "development", + "dependsOn": ["build"], + "options": { + "buildTarget": "research-service:build", + "runBuildTargetDependencies": false + }, + "configurations": { + "development": { + "buildTarget": "research-service:build:development" + }, + "production": { + "buildTarget": "research-service:build:production" + } + } + } + } +} diff --git a/apps/research-service/src/app.module.ts b/apps/research-service/src/app.module.ts new file mode 100644 index 0000000..b543cb5 --- /dev/null +++ b/apps/research-service/src/app.module.ts @@ -0,0 +1,11 @@ +import { Module } from '@nestjs/common'; +import { DatabaseModule } from '@terramatch-microservices/database'; +import { CommonModule } from '@terramatch-microservices/common'; +import { HealthModule } from './health/health.module'; + +@Module({ + imports: [DatabaseModule, CommonModule, HealthModule], + controllers: [], + providers: [], +}) +export class AppModule {} diff --git a/apps/research-service/src/assets/.gitkeep b/apps/research-service/src/assets/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/apps/research-service/src/health/health.controller.ts b/apps/research-service/src/health/health.controller.ts new file mode 100644 index 0000000..12ffbbb --- /dev/null +++ b/apps/research-service/src/health/health.controller.ts @@ -0,0 +1,32 @@ +import { Controller, Get } from '@nestjs/common'; +import { + HealthCheck, + HealthCheckService, + SequelizeHealthIndicator, +} from '@nestjs/terminus'; +import { NoBearerAuth } from '@terramatch-microservices/common/guards'; +import { ApiExcludeController } from '@nestjs/swagger'; +import { User } from '@terramatch-microservices/database/entities'; + +@Controller('health') +@ApiExcludeController() +export class HealthController { + constructor( + private readonly health: HealthCheckService, + private readonly db: SequelizeHealthIndicator + ) {} + + @Get() + @HealthCheck() + @NoBearerAuth + async check() { + const connection = await User.sequelize.connectionManager.getConnection({ type: 'read' }); + try { + return this.health.check([ + () => this.db.pingCheck('database', { connection }), + ]); + } finally { + User.sequelize.connectionManager.releaseConnection(connection); + } + } +} diff --git a/apps/research-service/src/health/health.module.ts b/apps/research-service/src/health/health.module.ts new file mode 100644 index 0000000..0208ef7 --- /dev/null +++ b/apps/research-service/src/health/health.module.ts @@ -0,0 +1,9 @@ +import { Module } from '@nestjs/common'; +import { TerminusModule } from '@nestjs/terminus'; +import { HealthController } from './health.controller'; + +@Module({ + imports: [TerminusModule], + controllers: [HealthController], +}) +export class HealthModule {} diff --git a/apps/research-service/src/main.ts b/apps/research-service/src/main.ts new file mode 100644 index 0000000..455ac07 --- /dev/null +++ b/apps/research-service/src/main.ts @@ -0,0 +1,31 @@ +import { Logger, ValidationPipe } from '@nestjs/common'; +import { NestFactory } from '@nestjs/core'; + +import { AppModule } from './app.module'; +import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger'; +import { TMLogService } from '@terramatch-microservices/common/util/tm-log.service'; + +async function bootstrap() { + const app = await NestFactory.create(AppModule); + + const config = new DocumentBuilder() + .setTitle('TerraMatch Research Service') + .setDescription('APIs related to needs for the data research team.') + .setVersion('1.0') + .addTag('research-service') + .build(); + const document = SwaggerModule.createDocument(app, config); + SwaggerModule.setup('research-service/documentation/api', app, document); + + app.useGlobalPipes(new ValidationPipe()); + app.useLogger(app.get(TMLogService)); + + const port = process.env.NODE_ENV === 'production' + ? 80 + : process.env.RESEARCH_SERVICE_PROXY_PORT ?? 4030; + await app.listen(port); + + Logger.log(`TerraMatch Research Service is running on: http://localhost:${port}`); +} + +bootstrap(); diff --git a/apps/research-service/tsconfig.app.json b/apps/research-service/tsconfig.app.json new file mode 100644 index 0000000..a2ce765 --- /dev/null +++ b/apps/research-service/tsconfig.app.json @@ -0,0 +1,12 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "../../dist/out-tsc", + "module": "commonjs", + "types": ["node"], + "emitDecoratorMetadata": true, + "target": "es2021" + }, + "exclude": ["jest.config.ts", "src/**/*.spec.ts", "src/**/*.test.ts"], + "include": ["src/**/*.ts"] +} diff --git a/apps/research-service/tsconfig.json b/apps/research-service/tsconfig.json new file mode 100644 index 0000000..c1e2dd4 --- /dev/null +++ b/apps/research-service/tsconfig.json @@ -0,0 +1,16 @@ +{ + "extends": "../../tsconfig.base.json", + "files": [], + "include": [], + "references": [ + { + "path": "./tsconfig.app.json" + }, + { + "path": "./tsconfig.spec.json" + } + ], + "compilerOptions": { + "esModuleInterop": true + } +} diff --git a/apps/research-service/tsconfig.spec.json b/apps/research-service/tsconfig.spec.json new file mode 100644 index 0000000..9b2a121 --- /dev/null +++ b/apps/research-service/tsconfig.spec.json @@ -0,0 +1,14 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "../../dist/out-tsc", + "module": "commonjs", + "types": ["jest", "node"] + }, + "include": [ + "jest.config.ts", + "src/**/*.test.ts", + "src/**/*.spec.ts", + "src/**/*.d.ts" + ] +} diff --git a/apps/research-service/webpack.config.js b/apps/research-service/webpack.config.js new file mode 100644 index 0000000..316ab88 --- /dev/null +++ b/apps/research-service/webpack.config.js @@ -0,0 +1,20 @@ +const { NxAppWebpackPlugin } = require('@nx/webpack/app-plugin'); +const { join } = require('path'); + +module.exports = { + output: { + path: join(__dirname, '../../dist/apps/research-service'), + }, + plugins: [ + new NxAppWebpackPlugin({ + target: 'node', + compiler: 'tsc', + main: './src/main.ts', + tsConfig: './tsconfig.app.json', + assets: ['./src/assets'], + optimization: false, + outputHashing: 'none', + generatePackageJson: true, + }), + ], +}; diff --git a/nx.json b/nx.json index c6f0e37..d11a75f 100644 --- a/nx.json +++ b/nx.json @@ -37,7 +37,8 @@ }, "exclude": [ "apps/user-service-e2e/**/*", - "apps/job-service-e2e/**/*" + "apps/job-service-e2e/**/*", + "apps/research-service-e2e/**/*" ] } ] From 797d479a47914d988d6124955e57c37ab15ea83c Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Thu, 24 Oct 2024 16:38:43 -0700 Subject: [PATCH 02/41] [TM-1249] Include a note about running many services at once. --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 5ddccd5..15f297b 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,9 @@ Repository for the Microservices API backend of the TerraMatch service an environment variable when building the api gateway: `ARCH=X86 nx build api-gateway`. * To run all services: * `nx run-many -t serve` + * The default maximum number of services it can run in parallel is 3. To run all of the services at once, use something like + `nx run-many --parallel=100 -t serve`, or you can cherry-pick which services you want to run instead with + `nx run-many -t serve --projects api-gateway user-service`. * Note: the first time this runs, the gateway will take quite awhile to start. It'll be faster on subsequent starts. * This starts the ApiGateway and all registered NX apps. * The apps will hot reload if their code, or any of their dependent code in libs changes. From 8be275482063c686f5216858b79f1986eadf276c Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Thu, 24 Oct 2024 17:04:38 -0700 Subject: [PATCH 03/41] [TM-1249] WIP polygon index endpoint definition. --- apps/research-service/src/app.module.ts | 3 +- .../dto/site-polygon-query.dto.ts | 13 +++ .../src/site-polygons/dto/site-polygon.dto.ts | 109 ++++++++++++++++++ .../site-polygons/site-polygons.controller.ts | 26 +++++ .../decorators/json-api-response.decorator.ts | 29 ++++- 5 files changed, 174 insertions(+), 6 deletions(-) create mode 100644 apps/research-service/src/site-polygons/dto/site-polygon-query.dto.ts create mode 100644 apps/research-service/src/site-polygons/dto/site-polygon.dto.ts create mode 100644 apps/research-service/src/site-polygons/site-polygons.controller.ts diff --git a/apps/research-service/src/app.module.ts b/apps/research-service/src/app.module.ts index b543cb5..04074eb 100644 --- a/apps/research-service/src/app.module.ts +++ b/apps/research-service/src/app.module.ts @@ -2,10 +2,11 @@ import { Module } from '@nestjs/common'; import { DatabaseModule } from '@terramatch-microservices/database'; import { CommonModule } from '@terramatch-microservices/common'; import { HealthModule } from './health/health.module'; +import { SitePolygonsController } from './site-polygons/site-polygons.controller'; @Module({ imports: [DatabaseModule, CommonModule, HealthModule], - controllers: [], + controllers: [SitePolygonsController], providers: [], }) export class AppModule {} diff --git a/apps/research-service/src/site-polygons/dto/site-polygon-query.dto.ts b/apps/research-service/src/site-polygons/dto/site-polygon-query.dto.ts new file mode 100644 index 0000000..32353f2 --- /dev/null +++ b/apps/research-service/src/site-polygons/dto/site-polygon-query.dto.ts @@ -0,0 +1,13 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { POLYGON_STATUSES, PolygonStatus } from './site-polygon.dto'; + +export class SitePolygonQueryDto { + @ApiProperty({ + enum: POLYGON_STATUSES, + name: 'polygonStatus[]', + required: false, + isArray: true, + description: 'Filter results by polygon status' + }) + polygonStatus?: PolygonStatus[] +} diff --git a/apps/research-service/src/site-polygons/dto/site-polygon.dto.ts b/apps/research-service/src/site-polygons/dto/site-polygon.dto.ts new file mode 100644 index 0000000..8d90655 --- /dev/null +++ b/apps/research-service/src/site-polygons/dto/site-polygon.dto.ts @@ -0,0 +1,109 @@ +import { JsonApiAttributes } from '@terramatch-microservices/common/dto/json-api-attributes'; +import { JsonApiDto } from '@terramatch-microservices/common/decorators'; +import { ApiProperty } from '@nestjs/swagger'; + +class TreeSpecies { + @ApiProperty({ example: 'Acacia binervia' }) + name: string; + + @ApiProperty({ example: 15000 }) + amount: number; +} + +class ReportingPeriod { + @ApiProperty() + dueAt: Date; + + @ApiProperty() + submittedAt: Date; + + @ApiProperty({ + type: () => TreeSpecies, + isArray: true, + description: 'The tree species reported as planted during this reporting period' + }) + treeSpecies: TreeSpecies[]; +} + +/** + * Note: this is required to be in the same order as on the source of truth in + * confluence: https://gfw.atlassian.net/wiki/spaces/TerraMatch/pages/1018396676/D.+Code+Criteria+and+Indicator+Tables#code_indicator + */ +const INDICATOR_TYPES = [ + 'treeCover', + 'treeCoverLoss', + 'treeCoverLossFires', + 'restorationEcoregion', + 'restorationIntervention', + 'treeCount', +]; +type IndicatorType = (typeof INDICATOR_TYPES)[number]; +class Indicator { + @ApiProperty({ enum: INDICATOR_TYPES }) + type: IndicatorType; + + @ApiProperty() + value: number; +} + +export const POLYGON_STATUSES = [ + 'draft', + 'submitted', + 'needs-more-information', + 'approved' +]; +export type PolygonStatus = (typeof POLYGON_STATUSES)[number]; + +@JsonApiDto({ type: 'sitePolygons' }) +export class SitePolygonDto extends JsonApiAttributes { + @ApiProperty() + name: string; + + @ApiProperty({ enum: POLYGON_STATUSES }) + status: PolygonStatus; + + @ApiProperty() + siteId: string; + + @ApiProperty() + plantStart: Date; + + @ApiProperty() + plantEnd: Date; + + @ApiProperty() + practice: string; + + @ApiProperty() + targetSys: string; + + @ApiProperty() + distr: string; + + @ApiProperty() + numTrees: number; + + @ApiProperty() + calcArea: number; + + @ApiProperty({ + type: () => Indicator, + isArray: true, + description: 'All indicators currently recorded for this site polygon' + }) + indicators: Indicator[]; + + @ApiProperty({ + type: () => TreeSpecies, + isArray: true, + description: 'The tree species associated with the establishment of the site that this polygon relates to.' + }) + establishmentTreeSpecies: TreeSpecies[]; + + @ApiProperty({ + type: () => ReportingPeriod, + isArray: true, + description: 'Access to reported trees planted for each approved report on this site.' + }) + reportingPeriods: ReportingPeriod[]; +} diff --git a/apps/research-service/src/site-polygons/site-polygons.controller.ts b/apps/research-service/src/site-polygons/site-polygons.controller.ts new file mode 100644 index 0000000..f1ec12b --- /dev/null +++ b/apps/research-service/src/site-polygons/site-polygons.controller.ts @@ -0,0 +1,26 @@ +import { + Controller, + Get, + NotImplementedException, + Query, + UnauthorizedException +} from '@nestjs/common'; +import { JsonApiDocument } from '@terramatch-microservices/common/util'; +import { ApiOperation } from '@nestjs/swagger'; +import { ApiException } from '@nanogiants/nestjs-swagger-api-exception-decorator'; +import { JsonApiResponse } from '@terramatch-microservices/common/decorators'; +import { SitePolygonDto } from './dto/site-polygon.dto'; +import { SitePolygonQueryDto } from './dto/site-polygon-query.dto'; + +@Controller('research/v3/sitePolygons') +export class SitePolygonsController { + @Get() + @ApiOperation({ operationId: 'sitePolygonsIndex', summary: 'Get all site polygons' }) + @JsonApiResponse({ data: { type: SitePolygonDto }, hasMany: true }) + @ApiException(() => UnauthorizedException, { + description: 'Authentication failed.' + }) + async findMany(@Query() query?: SitePolygonQueryDto): Promise { + throw new NotImplementedException(); + } +} diff --git a/libs/common/src/lib/decorators/json-api-response.decorator.ts b/libs/common/src/lib/decorators/json-api-response.decorator.ts index 735904a..2ef6c44 100644 --- a/libs/common/src/lib/decorators/json-api-response.decorator.ts +++ b/libs/common/src/lib/decorators/json-api-response.decorator.ts @@ -105,12 +105,23 @@ type Relationship = { } type Resource = { + /** + * The DTO for the attributes of the resource type + */ type: ResourceType; + relationships?: Relationship[]; } type JsonApiResponseProps = { data: Resource; + + /** + * Set to true if this endpoint returns more than one resource in the main `data` member. + * @default false + */ + hasMany?: boolean; + included?: Resource[]; } @@ -122,14 +133,22 @@ type JsonApiResponseProps = { export function JsonApiResponse( options: ApiResponseOptions & JsonApiResponseProps ) { - const { data, included, status, ...rest } = options; + const { data, hasMany, included, status, ...rest } = options; const extraModels: ResourceType[] = [data.type]; const document = { - data: { - type: "object", - properties: constructResource(data) - } + data: hasMany + ? { + type: "array", + items: { + type: "object", + properties: constructResource(data) + } + } + : { + type: "object", + properties: constructResource(data) + } } as { data: any; included?: any } if (included != null && included.length > 0) { for (const includedResource of included) { From 39e91c3925bbfd97cc267a84e5bf324d36f755a3 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Fri, 25 Oct 2024 11:04:53 -0700 Subject: [PATCH 04/41] [TM-1249] Add all filter options. --- .../dto/site-polygon-query.dto.ts | 36 ++++++++++++++++++- .../src/site-polygons/dto/site-polygon.dto.ts | 4 +-- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/apps/research-service/src/site-polygons/dto/site-polygon-query.dto.ts b/apps/research-service/src/site-polygons/dto/site-polygon-query.dto.ts index 32353f2..eb629b4 100644 --- a/apps/research-service/src/site-polygons/dto/site-polygon-query.dto.ts +++ b/apps/research-service/src/site-polygons/dto/site-polygon-query.dto.ts @@ -1,5 +1,10 @@ import { ApiProperty } from '@nestjs/swagger'; -import { POLYGON_STATUSES, PolygonStatus } from './site-polygon.dto'; +import { + INDICATOR_TYPES, + IndicatorType, + POLYGON_STATUSES, + PolygonStatus +} from './site-polygon.dto'; export class SitePolygonQueryDto { @ApiProperty({ @@ -10,4 +15,33 @@ export class SitePolygonQueryDto { description: 'Filter results by polygon status' }) polygonStatus?: PolygonStatus[] + + @ApiProperty({ + name: 'projectId[]', + required: false, + isArray: true, + description: 'Filter results by project UUID(s)' + }) + projectId?: string[] + + @ApiProperty({ + enum: INDICATOR_TYPES, + name: 'missingIndicator[]', + required: false, + isArray: true, + description: 'Filter results by polygons that are missing at least one of the indicators listed' + }) + missingIndicator?: IndicatorType[] + + @ApiProperty({ + required: false, + description: 'Filter results by polygons that have been modified since the date provided' + }) + lastModifiedDate?: Date + + @ApiProperty({ + required: false, + description: 'Filter results by polygons that are within the boundary of the polygon referenced by this UUID' + }) + boundaryPolygon?: string } diff --git a/apps/research-service/src/site-polygons/dto/site-polygon.dto.ts b/apps/research-service/src/site-polygons/dto/site-polygon.dto.ts index 8d90655..1383592 100644 --- a/apps/research-service/src/site-polygons/dto/site-polygon.dto.ts +++ b/apps/research-service/src/site-polygons/dto/site-polygon.dto.ts @@ -29,7 +29,7 @@ class ReportingPeriod { * Note: this is required to be in the same order as on the source of truth in * confluence: https://gfw.atlassian.net/wiki/spaces/TerraMatch/pages/1018396676/D.+Code+Criteria+and+Indicator+Tables#code_indicator */ -const INDICATOR_TYPES = [ +export const INDICATOR_TYPES = [ 'treeCover', 'treeCoverLoss', 'treeCoverLossFires', @@ -37,7 +37,7 @@ const INDICATOR_TYPES = [ 'restorationIntervention', 'treeCount', ]; -type IndicatorType = (typeof INDICATOR_TYPES)[number]; +export type IndicatorType = (typeof INDICATOR_TYPES)[number]; class Indicator { @ApiProperty({ enum: INDICATOR_TYPES }) type: IndicatorType; From a95b94b8389755e3bb210e279a30e0cba921ba07 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Fri, 25 Oct 2024 11:18:24 -0700 Subject: [PATCH 05/41] [TM-1249] Add pagination --- .../dto/site-polygon-query.dto.ts | 17 ++++++++++ .../site-polygons/site-polygons.controller.ts | 8 ++--- .../decorators/json-api-response.decorator.ts | 31 +++++++++++++++++-- 3 files changed, 50 insertions(+), 6 deletions(-) diff --git a/apps/research-service/src/site-polygons/dto/site-polygon-query.dto.ts b/apps/research-service/src/site-polygons/dto/site-polygon-query.dto.ts index eb629b4..9ad72f3 100644 --- a/apps/research-service/src/site-polygons/dto/site-polygon-query.dto.ts +++ b/apps/research-service/src/site-polygons/dto/site-polygon-query.dto.ts @@ -44,4 +44,21 @@ export class SitePolygonQueryDto { description: 'Filter results by polygons that are within the boundary of the polygon referenced by this UUID' }) boundaryPolygon?: string + + @ApiProperty({ + required: false, + name: 'page[size]', + description: 'The size of page being requested', + minimum: 1, + maximum: 100, + default: 100 + }) + pageSize?: number; + + @ApiProperty({ + required: false, + name: 'page[after]', + description: 'The last record before the page being requested. The value is a polygon UUID. If not provided, the first page is returned.' + }) + pageAfterCursor?: string; } diff --git a/apps/research-service/src/site-polygons/site-polygons.controller.ts b/apps/research-service/src/site-polygons/site-polygons.controller.ts index f1ec12b..5312122 100644 --- a/apps/research-service/src/site-polygons/site-polygons.controller.ts +++ b/apps/research-service/src/site-polygons/site-polygons.controller.ts @@ -1,4 +1,5 @@ import { + BadRequestException, Controller, Get, NotImplementedException, @@ -16,10 +17,9 @@ import { SitePolygonQueryDto } from './dto/site-polygon-query.dto'; export class SitePolygonsController { @Get() @ApiOperation({ operationId: 'sitePolygonsIndex', summary: 'Get all site polygons' }) - @JsonApiResponse({ data: { type: SitePolygonDto }, hasMany: true }) - @ApiException(() => UnauthorizedException, { - description: 'Authentication failed.' - }) + @JsonApiResponse({ data: { type: SitePolygonDto }, hasMany: true, pagination: true }) + @ApiException(() => UnauthorizedException, { description: 'Authentication failed.' }) + @ApiException(() => BadRequestException, { description: 'Pagination values are invalid.' }) async findMany(@Query() query?: SitePolygonQueryDto): Promise { throw new NotImplementedException(); } diff --git a/libs/common/src/lib/decorators/json-api-response.decorator.ts b/libs/common/src/lib/decorators/json-api-response.decorator.ts index 2ef6c44..dc2ba53 100644 --- a/libs/common/src/lib/decorators/json-api-response.decorator.ts +++ b/libs/common/src/lib/decorators/json-api-response.decorator.ts @@ -84,6 +84,11 @@ function constructResource(resource: Resource) { return def; } +function addMeta (document: Document, name: string, definition: any) { + if (document.meta == null) document.meta = { type: "object", properties: {} }; + document.meta.properties[name] = definition; +} + type ResourceType = new (...props: any[]) => JsonApiAttributes; type Relationship = { @@ -122,9 +127,20 @@ type JsonApiResponseProps = { */ hasMany?: boolean; + /** + * Set to true if this endpoint response documentation should include cursor pagination metadata. + */ + pagination?: boolean; + included?: Resource[]; } +type Document = { + data: any; + meta?: any; + included?: any; +} + /** * Decorator to simplify wrapping the response type from a controller method with the JSON API * response structure. Builds the JSON:API document structure and applies the ApiExtraModels and @@ -133,7 +149,7 @@ type JsonApiResponseProps = { export function JsonApiResponse( options: ApiResponseOptions & JsonApiResponseProps ) { - const { data, hasMany, included, status, ...rest } = options; + const { data, hasMany, pagination, included, status, ...rest } = options; const extraModels: ResourceType[] = [data.type]; const document = { @@ -149,7 +165,8 @@ export function JsonApiResponse( type: "object", properties: constructResource(data) } - } as { data: any; included?: any } + } as Document; + if (included != null && included.length > 0) { for (const includedResource of included) { extraModels.push(includedResource.type); @@ -168,6 +185,16 @@ export function JsonApiResponse( } } + if (pagination) { + addMeta(document, 'page', { + type: "object", + properties: { + cursor: { type: "string", description: "The cursor for the first record on this page." }, + total: { type: "number", description: "The total number of records on this page.", example: 42 } + } + }); + } + const apiResponseOptions = { ...rest, status: status ?? HttpStatus.OK, From f75bd5056229986036ecb7b742af98946ba77f31 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Fri, 25 Oct 2024 13:21:19 -0700 Subject: [PATCH 06/41] [TM-1249] Document all the indicator types. --- .../src/site-polygons/dto/indicators.dto.ts | 133 ++++++++++++++++++ .../dto/site-polygon-query.dto.ts | 38 ++--- .../src/site-polygons/dto/site-polygon.dto.ts | 49 +++---- .../site-polygons/site-polygons.controller.ts | 43 +++++- 4 files changed, 213 insertions(+), 50 deletions(-) create mode 100644 apps/research-service/src/site-polygons/dto/indicators.dto.ts diff --git a/apps/research-service/src/site-polygons/dto/indicators.dto.ts b/apps/research-service/src/site-polygons/dto/indicators.dto.ts new file mode 100644 index 0000000..6912692 --- /dev/null +++ b/apps/research-service/src/site-polygons/dto/indicators.dto.ts @@ -0,0 +1,133 @@ +import { ApiProperty } from '@nestjs/swagger'; + +// Matches the indicators defined on https://gfw.atlassian.net/wiki/spaces/TerraMatch/pages/1469448210/Indicator+Data+Model +export const INDICATORS = { + 1: 'treeCover', + 2: 'treeCoverLoss', + 3: 'treeCoverLossFires', + 4: 'restorationByEcoRegion', + 5: 'restorationByStrategy', + 6: 'restorationByLandUse', + 7: 'treeCount', + 8: 'earlyTreeVerification', + 9: 'fieldMonitoring', + 10: 'msuCarbon' +} as const; +export const INDICATOR_SLUGS = Object.values(INDICATORS); +export type IndicatorSlug = (typeof INDICATOR_SLUGS)[number]; + +export class IndicatorTreeCoverLossDto { + @ApiProperty({ enum: [INDICATORS[2], INDICATORS[3]] }) + indicatorSlug: typeof INDICATORS[2] | typeof INDICATORS[3] + + @ApiProperty({ example: '2024' }) + yearOfAnalysis: number; + + @ApiProperty({ + type: 'object', + description: 'Mapping of year of analysis to value.', + example: { 2024: '0.6', 2023: '0.5' } + }) + value: Record; +} + +export class IndicatorHectaresDto { + @ApiProperty({ enum: [INDICATORS[4], INDICATORS[5], INDICATORS[6]] }) + indicatorSlug: typeof INDICATORS[4] | typeof INDICATORS[5] | typeof INDICATORS[6]; + + @ApiProperty({ example: '2024' }) + yearOfAnalysis: number; + + @ApiProperty({ + type: 'object', + description: 'Mapping of area type (eco region, land use, etc) to hectares', + example: { 'Northern Acacia-Commiphora bushlands and thickets': 0.104 } + }) + value: Record; +} + +export class IndicatorTreeCountDto { + @ApiProperty({ enum: [INDICATORS[7], INDICATORS[8]] }) + indicatorSlug: typeof INDICATORS[7] | typeof INDICATORS[8]; + + @ApiProperty({ example: '2024' }) + yearOfAnalysis: number; + + @ApiProperty() + surveyType: string; + + @ApiProperty() + surveyId: number; + + @ApiProperty() + treeCount: number; + + @ApiProperty({ example: 'types TBD' }) + uncertaintyType: string; + + @ApiProperty() + imagerySource: string; + + @ApiProperty({ type: 'url' }) + imageryId: string; + + @ApiProperty() + projectPhase: string; + + @ApiProperty() + confidence: number; +} + +export class IndicatorTreeCoverDto { + @ApiProperty({ enum: [INDICATORS[1]] }) + indicatorSlug: typeof INDICATORS[1]; + + @ApiProperty({ example: '2024' }) + yearOfAnalysis: number; + + @ApiProperty({ example: '2024' }) + projectPhase: string; + + @ApiProperty() + percentCover: number; + + @ApiProperty() + plusMinusPercent: number +} + +export class IndicatorFieldMonitoringDto { + @ApiProperty({ enum: [INDICATORS[9]] }) + indicatorSlug: typeof INDICATORS[9]; + + @ApiProperty({ example: '2024' }) + yearOfAnalysis: number; + + @ApiProperty() + treeCount: number; + + @ApiProperty() + projectPhase: string; + + @ApiProperty() + species: string; + + @ApiProperty() + survivalRate: number; +} + +export class IndicatorMsuCarbonDto { + @ApiProperty({ enum: [INDICATORS[10]] }) + indicatorSlug: typeof INDICATORS[10]; + + @ApiProperty({ example: '2024' }) + yearOfAnalysis: number; + + @ApiProperty() + carbonOutput: number; + + @ApiProperty() + projectPhase: string; + + @ApiProperty() + confidence: number; +} diff --git a/apps/research-service/src/site-polygons/dto/site-polygon-query.dto.ts b/apps/research-service/src/site-polygons/dto/site-polygon-query.dto.ts index 9ad72f3..db6c42e 100644 --- a/apps/research-service/src/site-polygons/dto/site-polygon-query.dto.ts +++ b/apps/research-service/src/site-polygons/dto/site-polygon-query.dto.ts @@ -1,10 +1,6 @@ import { ApiProperty } from '@nestjs/swagger'; -import { - INDICATOR_TYPES, - IndicatorType, - POLYGON_STATUSES, - PolygonStatus -} from './site-polygon.dto'; +import { POLYGON_STATUSES, PolygonStatus } from './site-polygon.dto'; +import { INDICATOR_SLUGS, IndicatorSlug } from './indicators.dto'; export class SitePolygonQueryDto { @ApiProperty({ @@ -12,38 +8,41 @@ export class SitePolygonQueryDto { name: 'polygonStatus[]', required: false, isArray: true, - description: 'Filter results by polygon status' + description: 'Filter results by polygon status', }) - polygonStatus?: PolygonStatus[] + polygonStatus?: PolygonStatus[]; @ApiProperty({ name: 'projectId[]', required: false, isArray: true, - description: 'Filter results by project UUID(s)' + description: 'Filter results by project UUID(s)', }) - projectId?: string[] + projectId?: string[]; @ApiProperty({ - enum: INDICATOR_TYPES, + enum: INDICATOR_SLUGS, name: 'missingIndicator[]', required: false, isArray: true, - description: 'Filter results by polygons that are missing at least one of the indicators listed' + description: + 'Filter results by polygons that are missing at least one of the indicators listed', }) - missingIndicator?: IndicatorType[] + missingIndicator?: IndicatorSlug[]; @ApiProperty({ required: false, - description: 'Filter results by polygons that have been modified since the date provided' + description: + 'Filter results by polygons that have been modified since the date provided', }) - lastModifiedDate?: Date + lastModifiedDate?: Date; @ApiProperty({ required: false, - description: 'Filter results by polygons that are within the boundary of the polygon referenced by this UUID' + description: + 'Filter results by polygons that are within the boundary of the polygon referenced by this UUID', }) - boundaryPolygon?: string + boundaryPolygon?: string; @ApiProperty({ required: false, @@ -51,14 +50,15 @@ export class SitePolygonQueryDto { description: 'The size of page being requested', minimum: 1, maximum: 100, - default: 100 + default: 100, }) pageSize?: number; @ApiProperty({ required: false, name: 'page[after]', - description: 'The last record before the page being requested. The value is a polygon UUID. If not provided, the first page is returned.' + description: + 'The last record before the page being requested. The value is a polygon UUID. If not provided, the first page is returned.', }) pageAfterCursor?: string; } diff --git a/apps/research-service/src/site-polygons/dto/site-polygon.dto.ts b/apps/research-service/src/site-polygons/dto/site-polygon.dto.ts index 1383592..86164d7 100644 --- a/apps/research-service/src/site-polygons/dto/site-polygon.dto.ts +++ b/apps/research-service/src/site-polygons/dto/site-polygon.dto.ts @@ -1,6 +1,12 @@ import { JsonApiAttributes } from '@terramatch-microservices/common/dto/json-api-attributes'; import { JsonApiDto } from '@terramatch-microservices/common/decorators'; import { ApiProperty } from '@nestjs/swagger'; +import { + IndicatorFieldMonitoringDto, + IndicatorHectaresDto, IndicatorMsuCarbonDto, + IndicatorTreeCountDto, IndicatorTreeCoverDto, + IndicatorTreeCoverLossDto +} from './indicators.dto'; class TreeSpecies { @ApiProperty({ example: 'Acacia binervia' }) @@ -25,27 +31,6 @@ class ReportingPeriod { treeSpecies: TreeSpecies[]; } -/** - * Note: this is required to be in the same order as on the source of truth in - * confluence: https://gfw.atlassian.net/wiki/spaces/TerraMatch/pages/1018396676/D.+Code+Criteria+and+Indicator+Tables#code_indicator - */ -export const INDICATOR_TYPES = [ - 'treeCover', - 'treeCoverLoss', - 'treeCoverLossFires', - 'restorationEcoregion', - 'restorationIntervention', - 'treeCount', -]; -export type IndicatorType = (typeof INDICATOR_TYPES)[number]; -class Indicator { - @ApiProperty({ enum: INDICATOR_TYPES }) - type: IndicatorType; - - @ApiProperty() - value: number; -} - export const POLYGON_STATUSES = [ 'draft', 'submitted', @@ -87,11 +72,27 @@ export class SitePolygonDto extends JsonApiAttributes { calcArea: number; @ApiProperty({ - type: () => Indicator, - isArray: true, + type: 'array', + items: { + oneOf: [ + { $ref: '#/components/schemas/IndicatorTreeCoverLossDto' }, + { $ref: '#/components/schemas/IndicatorHectaresDto' }, + { $ref: '#/components/schemas/IndicatorTreeCountDto' }, + { $ref: '#/components/schemas/IndicatorTreeCoverDto' }, + { $ref: '#/components/schemas/IndicatorFieldMonitoringDto' }, + { $ref: '#/components/schemas/IndicatorMsuCarbonDto' }, + ] + }, description: 'All indicators currently recorded for this site polygon' }) - indicators: Indicator[]; + indicators: ( + IndicatorTreeCoverLossDto | + IndicatorHectaresDto | + IndicatorTreeCountDto | + IndicatorTreeCoverDto | + IndicatorFieldMonitoringDto | + IndicatorMsuCarbonDto + )[] @ApiProperty({ type: () => TreeSpecies, diff --git a/apps/research-service/src/site-polygons/site-polygons.controller.ts b/apps/research-service/src/site-polygons/site-polygons.controller.ts index 5312122..4a0529a 100644 --- a/apps/research-service/src/site-polygons/site-polygons.controller.ts +++ b/apps/research-service/src/site-polygons/site-polygons.controller.ts @@ -4,23 +4,52 @@ import { Get, NotImplementedException, Query, - UnauthorizedException + UnauthorizedException, } from '@nestjs/common'; import { JsonApiDocument } from '@terramatch-microservices/common/util'; -import { ApiOperation } from '@nestjs/swagger'; +import { ApiExtraModels, ApiOperation } from '@nestjs/swagger'; import { ApiException } from '@nanogiants/nestjs-swagger-api-exception-decorator'; import { JsonApiResponse } from '@terramatch-microservices/common/decorators'; import { SitePolygonDto } from './dto/site-polygon.dto'; import { SitePolygonQueryDto } from './dto/site-polygon-query.dto'; +import { + IndicatorFieldMonitoringDto, + IndicatorHectaresDto, + IndicatorMsuCarbonDto, + IndicatorTreeCountDto, + IndicatorTreeCoverDto, + IndicatorTreeCoverLossDto, +} from './dto/indicators.dto'; @Controller('research/v3/sitePolygons') +@ApiExtraModels( + IndicatorTreeCoverLossDto, + IndicatorHectaresDto, + IndicatorTreeCountDto, + IndicatorTreeCoverDto, + IndicatorFieldMonitoringDto, + IndicatorMsuCarbonDto +) export class SitePolygonsController { @Get() - @ApiOperation({ operationId: 'sitePolygonsIndex', summary: 'Get all site polygons' }) - @JsonApiResponse({ data: { type: SitePolygonDto }, hasMany: true, pagination: true }) - @ApiException(() => UnauthorizedException, { description: 'Authentication failed.' }) - @ApiException(() => BadRequestException, { description: 'Pagination values are invalid.' }) - async findMany(@Query() query?: SitePolygonQueryDto): Promise { + @ApiOperation({ + operationId: 'sitePolygonsIndex', + summary: 'Get all site polygons', + }) + @JsonApiResponse({ + data: { type: SitePolygonDto }, + hasMany: true, + pagination: true, + }) + @ApiException(() => UnauthorizedException, { + description: 'Authentication failed.', + }) + @ApiException(() => BadRequestException, { + description: 'Pagination values are invalid.', + }) + async findMany( + @Query() query?: SitePolygonQueryDto + ): Promise { throw new NotImplementedException(); } } From 1e13db212a01b56f17bc82e9e76c0ef40c52f321 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Fri, 25 Oct 2024 14:29:40 -0700 Subject: [PATCH 07/41] [TM-1249] Document update endpoint. --- .../dto/site-polygon-update.dto.ts | 48 +++++++++++++++++++ .../src/site-polygons/dto/site-polygon.dto.ts | 2 +- .../site-polygons/site-polygons.controller.ts | 45 +++++++++-------- 3 files changed, 75 insertions(+), 20 deletions(-) create mode 100644 apps/research-service/src/site-polygons/dto/site-polygon-update.dto.ts diff --git a/apps/research-service/src/site-polygons/dto/site-polygon-update.dto.ts b/apps/research-service/src/site-polygons/dto/site-polygon-update.dto.ts new file mode 100644 index 0000000..6e23a0d --- /dev/null +++ b/apps/research-service/src/site-polygons/dto/site-polygon-update.dto.ts @@ -0,0 +1,48 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { + IndicatorFieldMonitoringDto, + IndicatorHectaresDto, IndicatorMsuCarbonDto, + IndicatorTreeCountDto, IndicatorTreeCoverDto, + IndicatorTreeCoverLossDto +} from './indicators.dto'; + +class SitePolygonUpdateAttributes { + @ApiProperty({ + type: 'array', + items: { + oneOf: [ + { $ref: '#/components/schemas/IndicatorTreeCoverLossDto' }, + { $ref: '#/components/schemas/IndicatorHectaresDto' }, + { $ref: '#/components/schemas/IndicatorTreeCountDto' }, + { $ref: '#/components/schemas/IndicatorTreeCoverDto' }, + { $ref: '#/components/schemas/IndicatorFieldMonitoringDto' }, + { $ref: '#/components/schemas/IndicatorMsuCarbonDto' }, + ] + }, + description: 'All indicators to update for this polygon' + }) + indicators: ( + IndicatorTreeCoverLossDto | + IndicatorHectaresDto | + IndicatorTreeCountDto | + IndicatorTreeCoverDto | + IndicatorFieldMonitoringDto | + IndicatorMsuCarbonDto + )[]; +} + +class SitePolygonUpdate { + @ApiProperty({ enum: ['sitePolygons'] }) + type: 'sitePolygons'; + + @ApiProperty({ format: 'uuid' }) + id: string; + + @ApiProperty({ type: () => SitePolygonUpdateAttributes }) + attributes: SitePolygonUpdateAttributes; +} + +export class SitePolygonBulkUpdateBodyDto { + @ApiProperty({ isArray: true, type: () => SitePolygonUpdate }) + data: SitePolygonUpdate[]; +} diff --git a/apps/research-service/src/site-polygons/dto/site-polygon.dto.ts b/apps/research-service/src/site-polygons/dto/site-polygon.dto.ts index 86164d7..ed97766 100644 --- a/apps/research-service/src/site-polygons/dto/site-polygon.dto.ts +++ b/apps/research-service/src/site-polygons/dto/site-polygon.dto.ts @@ -92,7 +92,7 @@ export class SitePolygonDto extends JsonApiAttributes { IndicatorTreeCoverDto | IndicatorFieldMonitoringDto | IndicatorMsuCarbonDto - )[] + )[]; @ApiProperty({ type: () => TreeSpecies, diff --git a/apps/research-service/src/site-polygons/site-polygons.controller.ts b/apps/research-service/src/site-polygons/site-polygons.controller.ts index 4a0529a..da51b6d 100644 --- a/apps/research-service/src/site-polygons/site-polygons.controller.ts +++ b/apps/research-service/src/site-polygons/site-polygons.controller.ts @@ -1,13 +1,13 @@ import { - BadRequestException, + BadRequestException, Body, Controller, Get, - NotImplementedException, + NotImplementedException, Patch, Query, - UnauthorizedException, + UnauthorizedException } from '@nestjs/common'; import { JsonApiDocument } from '@terramatch-microservices/common/util'; -import { ApiExtraModels, ApiOperation } from '@nestjs/swagger'; +import { ApiExtraModels, ApiOkResponse, ApiOperation } from '@nestjs/swagger'; import { ApiException } from '@nanogiants/nestjs-swagger-api-exception-decorator'; import { JsonApiResponse } from '@terramatch-microservices/common/decorators'; import { SitePolygonDto } from './dto/site-polygon.dto'; @@ -20,6 +20,7 @@ import { IndicatorTreeCoverDto, IndicatorTreeCoverLossDto, } from './dto/indicators.dto'; +import { SitePolygonBulkUpdateBodyDto } from './dto/site-polygon-update.dto'; @Controller('research/v3/sitePolygons') @ApiExtraModels( @@ -32,24 +33,30 @@ import { ) export class SitePolygonsController { @Get() - @ApiOperation({ - operationId: 'sitePolygonsIndex', - summary: 'Get all site polygons', - }) - @JsonApiResponse({ - data: { type: SitePolygonDto }, - hasMany: true, - pagination: true, - }) - @ApiException(() => UnauthorizedException, { - description: 'Authentication failed.', - }) - @ApiException(() => BadRequestException, { - description: 'Pagination values are invalid.', - }) + @ApiOperation({ operationId: 'sitePolygonsIndex', summary: 'Get all site polygons' }) + @JsonApiResponse({ data: { type: SitePolygonDto }, hasMany: true, pagination: true }) + @ApiException(() => UnauthorizedException, { description: 'Authentication failed.' }) + @ApiException(() => BadRequestException, { description: 'Pagination values are invalid.' }) async findMany( @Query() query?: SitePolygonQueryDto ): Promise { throw new NotImplementedException(); } + + @Patch() + @ApiOperation({ + operationId: 'bulkUpdateSitePolygons', + summary: 'Update indicators for site polygons', + description: + `If an indicator is provided that already exists, it will be updated with the value in the + payload. If a new indicator is provided, it will be created in the DB. Indicators are keyed + off of the combination of site polygon ID, indicatorSlug, and yearOfAnalysis.` + }) + @ApiOkResponse() + @ApiException(() => UnauthorizedException, { description: 'Authentication failed.' }) + async bulkUpdate( + @Body() updatePayload: SitePolygonBulkUpdateBodyDto + ): Promise { + throw new NotImplementedException(); + } } From f8381473fd13a29b4e43b1fc405a2516f7805cb9 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Tue, 29 Oct 2024 19:24:28 -0700 Subject: [PATCH 08/41] [TM-1249] DOCKER_HOST isn't needed anymore. --- .env.local.sample | 2 -- 1 file changed, 2 deletions(-) diff --git a/.env.local.sample b/.env.local.sample index e46f951..1b49723 100644 --- a/.env.local.sample +++ b/.env.local.sample @@ -1,5 +1,3 @@ -DOCKER_HOST=unix://$HOME/.docker/run/docker.sock - USER_SERVICE_PORT=4010 JOB_SERVICE_PORT=4020 RESEARCH_SERVICE_PORT=4030 From 9bb1a4d13c529286fffec5263b047eb49451ac74 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Thu, 31 Oct 2024 13:57:25 -0700 Subject: [PATCH 09/41] [TM-1384] Include direct framework associations on users/me response. --- .../src/users/users.controller.ts | 7 ++- .../src/lib/entities/framework-user.entity.ts | 20 ++++++ libs/database/src/lib/entities/index.ts | 1 + libs/database/src/lib/entities/user.entity.ts | 63 +++++++++++++------ package-lock.json | 8 +++ package.json | 2 + 6 files changed, 79 insertions(+), 22 deletions(-) create mode 100644 libs/database/src/lib/entities/framework-user.entity.ts diff --git a/apps/user-service/src/users/users.controller.ts b/apps/user-service/src/users/users.controller.ts index 8d5ffb5..2cc64ce 100644 --- a/apps/user-service/src/users/users.controller.ts +++ b/apps/user-service/src/users/users.controller.ts @@ -53,13 +53,16 @@ export class UsersController { @Request() { authenticatedUserId } ): Promise { const userId = pathId === 'me' ? authenticatedUserId : parseInt(pathId); - const user = await User.findOne({ include: ['roles', 'organisation'], where: { id: userId }, }); + const user = await User.findOne({ + include: ['roles', 'organisation', 'frameworks'], + where: { id: userId }, + }); if (user == null) throw new NotFoundException(); await this.policyService.authorize('read', user); const document = buildJsonApi(); - const userResource = document.addData(user.uuid, new UserDto(user, await user.frameworks())); + const userResource = document.addData(user.uuid, new UserDto(user, await user.myFrameworks())); const org = await user.primaryOrganisation(); if (org != null) { diff --git a/libs/database/src/lib/entities/framework-user.entity.ts b/libs/database/src/lib/entities/framework-user.entity.ts new file mode 100644 index 0000000..ab6d5de --- /dev/null +++ b/libs/database/src/lib/entities/framework-user.entity.ts @@ -0,0 +1,20 @@ +import { AutoIncrement, Column, ForeignKey, Model, PrimaryKey, Table } from 'sequelize-typescript'; +import { BIGINT } from 'sequelize'; +import { Framework } from './framework.entity'; +import { User } from './user.entity'; + +@Table({ tableName: 'framework_user', underscored: true }) +export class FrameworkUser extends Model { + @PrimaryKey + @AutoIncrement + @Column(BIGINT.UNSIGNED) + override id: number; + + @ForeignKey(() => Framework) + @Column(BIGINT.UNSIGNED) + frameworkId: number; + + @ForeignKey(() => User) + @Column(BIGINT.UNSIGNED) + userId: number; +} diff --git a/libs/database/src/lib/entities/index.ts b/libs/database/src/lib/entities/index.ts index bb15fd4..0967a28 100644 --- a/libs/database/src/lib/entities/index.ts +++ b/libs/database/src/lib/entities/index.ts @@ -1,5 +1,6 @@ export * from './delayed-job.entity'; export * from './framework.entity'; +export * from './framework-user.entity'; export * from './model-has-role.entity' export * from './organisation.entity'; export * from './organisation-user.entity'; diff --git a/libs/database/src/lib/entities/user.entity.ts b/libs/database/src/lib/entities/user.entity.ts index 4e9bbd5..d795151 100644 --- a/libs/database/src/lib/entities/user.entity.ts +++ b/libs/database/src/lib/entities/user.entity.ts @@ -1,15 +1,17 @@ +import { uniq } from 'lodash'; import { AllowNull, AutoIncrement, BelongsTo, BelongsToMany, - Column, Default, + Column, + Default, ForeignKey, Index, Model, PrimaryKey, Table, - Unique + Unique, } from 'sequelize-typescript'; import { BIGINT, BOOLEAN, col, DATE, fn, Op, STRING, UUID } from 'sequelize'; import { Role } from './role.entity'; @@ -20,6 +22,7 @@ import { Project } from './project.entity'; import { ProjectUser } from './project-user.entity'; import { Organisation } from './organisation.entity'; import { OrganisationUser } from './organisation-user.entity'; +import { FrameworkUser } from './framework-user.entity'; @Table({ tableName: 'users', underscored: true, paranoid: true }) export class User extends Model { @@ -257,39 +260,59 @@ export class User extends Model { : this._primaryOrganisation; } - private _frameworks?: Framework[]; - async frameworks(): Promise { - if (this._frameworks == null) { + @BelongsToMany(() => Framework, () => FrameworkUser) + frameworks: Framework[]; + + async loadFrameworks() { + if (this.frameworks == null) { + this.frameworks = await (this as User).$get('frameworks'); + } + return this.frameworks; + } + + private _myFrameworks?: Framework[]; + async myFrameworks(): Promise { + if (this._myFrameworks == null) { await this.loadRoles(); const isAdmin = this.roles.find(({ name }) => name.startsWith('admin-')) != null; - let frameworkSlugs: string[]; + await this.loadFrameworks(); + + let frameworkSlugs: string[] = this.frameworks.map(({ slug }) => slug); if (isAdmin) { // Admins have access to all frameworks their permissions say they do const permissions = await Permission.getUserPermissionNames(this.id); const prefix = 'framework-'; - frameworkSlugs = permissions - .filter((permission) => permission.startsWith(prefix)) - .map((permission) => permission.substring(prefix.length)); + frameworkSlugs = [ + ...frameworkSlugs, + ...permissions + .filter((permission) => permission.startsWith(prefix)) + .map((permission) => permission.substring(prefix.length)), + ]; } else { // Other users have access to the frameworks embodied by their set of projects - frameworkSlugs = ( - await (this as User).$get('projects', { - attributes: [ - [fn('DISTINCT', col('Project.framework_key')), 'frameworkKey'], - ], - raw: true, - }) - ).map(({ frameworkKey }) => frameworkKey); + frameworkSlugs = [ + ...frameworkSlugs, + ...( + await (this as User).$get('projects', { + attributes: [ + [fn('DISTINCT', col('Project.framework_key')), 'frameworkKey'], + ], + raw: true, + }) + ).map(({ frameworkKey }) => frameworkKey), + ]; } - if (frameworkSlugs.length == 0) return (this._frameworks = []); - return (this._frameworks = await Framework.findAll({ + if (frameworkSlugs.length == 0) return (this._myFrameworks = []); + + frameworkSlugs = uniq(frameworkSlugs); + return (this._myFrameworks = await Framework.findAll({ where: { slug: { [Op.in]: frameworkSlugs } }, })); } - return this._frameworks; + return this._myFrameworks; } } diff --git a/package-lock.json b/package-lock.json index 197a44c..f4b0b90 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,6 +23,7 @@ "bcryptjs": "^2.4.3", "class-transformer": "^0.5.1", "class-validator": "^0.14.1", + "lodash": "^4.17.21", "mariadb": "^3.3.2", "mysql2": "^3.11.2", "nestjs-request-context": "^3.0.0", @@ -51,6 +52,7 @@ "@swc/helpers": "~0.5.11", "@types/bcryptjs": "^2.4.6", "@types/jest": "^29.5.12", + "@types/lodash": "^4.17.13", "@types/node": "~18.16.9", "@types/supertest": "^6.0.2", "@types/validator": "^13.12.2", @@ -4685,6 +4687,12 @@ "@types/node": "*" } }, + "node_modules/@types/lodash": { + "version": "4.17.13", + "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.13.tgz", + "integrity": "sha512-lfx+dftrEZcdBPczf9d0Qv0x+j/rfNCMuC6OcfXmO8gkfeNAY88PgKUbvG56whcN23gc27yenwF6oJZXGFpYxg==", + "dev": true + }, "node_modules/@types/methods": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/@types/methods/-/methods-1.1.4.tgz", diff --git a/package.json b/package.json index 1252ebf..48c2e49 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "bcryptjs": "^2.4.3", "class-transformer": "^0.5.1", "class-validator": "^0.14.1", + "lodash": "^4.17.21", "mariadb": "^3.3.2", "mysql2": "^3.11.2", "nestjs-request-context": "^3.0.0", @@ -47,6 +48,7 @@ "@swc/helpers": "~0.5.11", "@types/bcryptjs": "^2.4.6", "@types/jest": "^29.5.12", + "@types/lodash": "^4.17.13", "@types/node": "~18.16.9", "@types/supertest": "^6.0.2", "@types/validator": "^13.12.2", From 176fecb617b21613a3fb0ab1ec05d76617448f0a Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Tue, 5 Nov 2024 15:55:37 -0800 Subject: [PATCH 10/41] [TM-1451] Add npm scripts for typical start configurations. --- package.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 48c2e49..c1a4189 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,11 @@ "name": "@terramatch-microservices/source", "version": "0.0.0", "license": "MIT", - "scripts": {}, + "scripts": { + "research": "nx run-many -t serve --projects research-service", + "fe-services": "nx run-many -t serve --projects user-service job-service", + "all": "nx run-many --parallel=100 -t serve" + }, "private": true, "dependencies": { "@casl/ability": "^6.7.1", From 66e7c54aa1a12d18163cc3c4e220bb94947f37f2 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Tue, 5 Nov 2024 16:51:46 -0800 Subject: [PATCH 11/41] [TM-1451] Match prettier config from the FE codebase. --- .prettierrc | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.prettierrc b/.prettierrc index 544138b..d78d400 100644 --- a/.prettierrc +++ b/.prettierrc @@ -1,3 +1,11 @@ { - "singleQuote": true + "arrowParens": "avoid", + "bracketSpacing": true, + "jsxBracketSameLine": false, + "printWidth": 120, + "semi": true, + "singleQuote": false, + "tabWidth": 2, + "trailingComma": "none", + "useTabs": false } From de79a7cb06cb53e34839a2e4f9518bc1dd1aad67 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Tue, 5 Nov 2024 17:07:51 -0800 Subject: [PATCH 12/41] [TM-1451] Improve validation --- .../src/jobs/dto/delayed-job.dto.ts | 20 ++--- apps/job-service/src/main.ts | 28 +++---- apps/research-service/src/app.module.ts | 3 +- apps/research-service/src/main.ts | 26 +++--- .../dto/site-polygon-query.dto.ts | 84 ++++++++++--------- .../site-polygons/site-polygons.controller.ts | 66 ++++++++------- .../site-polygons/site-polygons.service.ts | 12 +++ apps/user-service/src/main.ts | 28 +++---- 8 files changed, 146 insertions(+), 121 deletions(-) create mode 100644 apps/research-service/src/site-polygons/site-polygons.service.ts diff --git a/apps/job-service/src/jobs/dto/delayed-job.dto.ts b/apps/job-service/src/jobs/dto/delayed-job.dto.ts index 6874ef4..91b7ba6 100644 --- a/apps/job-service/src/jobs/dto/delayed-job.dto.ts +++ b/apps/job-service/src/jobs/dto/delayed-job.dto.ts @@ -1,13 +1,12 @@ -import { JsonApiAttributes } from '@terramatch-microservices/common/dto/json-api-attributes'; -import { JsonApiDto } from '@terramatch-microservices/common/decorators'; -import { ApiProperty } from '@nestjs/swagger'; -import { DelayedJob } from '@terramatch-microservices/database/entities'; -import { JSON } from 'sequelize'; +import { JsonApiAttributes } from "@terramatch-microservices/common/dto/json-api-attributes"; +import { JsonApiDto } from "@terramatch-microservices/common/decorators"; +import { ApiProperty } from "@nestjs/swagger"; +import { DelayedJob } from "@terramatch-microservices/database/entities"; -const STATUSES = ['pending', 'failed', 'succeeded'] +const STATUSES = ["pending", "failed", "succeeded"]; type Status = (typeof STATUSES)[number]; -@JsonApiDto({ type: 'delayedJobs' }) +@JsonApiDto({ type: "delayedJobs" }) export class DelayedJobDto extends JsonApiAttributes { constructor(job: DelayedJob) { const { status, statusCode, payload } = job; @@ -15,19 +14,20 @@ export class DelayedJobDto extends JsonApiAttributes { } @ApiProperty({ - description: 'The current status of the job. If the status is not pending, the payload and statusCode will be provided.', + description: + "The current status of the job. If the status is not pending, the payload and statusCode will be provided.", enum: STATUSES }) status: Status; @ApiProperty({ - description: 'If the job is out of pending state, this is the HTTP status code for the completed process', + description: "If the job is out of pending state, this is the HTTP status code for the completed process", nullable: true }) statusCode: number | null; @ApiProperty({ - description: 'If the job is out of pending state, this is the JSON payload for the completed process', + description: "If the job is out of pending state, this is the JSON payload for the completed process", nullable: true }) payload: object | null; diff --git a/apps/job-service/src/main.ts b/apps/job-service/src/main.ts index cee0562..c6398b6 100644 --- a/apps/job-service/src/main.ts +++ b/apps/job-service/src/main.ts @@ -1,33 +1,31 @@ -import { Logger, ValidationPipe } from '@nestjs/common'; -import { NestFactory } from '@nestjs/core'; +import { Logger, ValidationPipe } from "@nestjs/common"; +import { NestFactory } from "@nestjs/core"; -import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger'; -import { TMLogService } from '@terramatch-microservices/common/util/tm-log.service'; -import { AppModule } from './app.module'; +import { DocumentBuilder, SwaggerModule } from "@nestjs/swagger"; +import { TMLogService } from "@terramatch-microservices/common/util/tm-log.service"; +import { AppModule } from "./app.module"; async function bootstrap() { const app = await NestFactory.create(AppModule); - if (process.env.NODE_ENV === 'development') { + if (process.env.NODE_ENV === "development") { // CORS is handled by the Api Gateway in AWS app.enableCors(); } const config = new DocumentBuilder() - .setTitle('TerraMatch Job Service') - .setDescription('APIs related to delayed jobs') - .setVersion('1.0') - .addTag('job-service') + .setTitle("TerraMatch Job Service") + .setDescription("APIs related to delayed jobs") + .setVersion("1.0") + .addTag("job-service") .build(); const document = SwaggerModule.createDocument(app, config); - SwaggerModule.setup('job-service/documentation/api', app, document); + SwaggerModule.setup("job-service/documentation/api", app, document); - app.useGlobalPipes(new ValidationPipe()); + app.useGlobalPipes(new ValidationPipe({ transform: true, transformOptions: { enableImplicitConversion: true } })); app.useLogger(app.get(TMLogService)); - const port = process.env.NODE_ENV === 'production' - ? 80 - : process.env.JOB_SERVICE_PORT ?? 4020; + const port = process.env.NODE_ENV === "production" ? 80 : process.env.JOB_SERVICE_PORT ?? 4020; await app.listen(port); Logger.log(`TerraMatch Job Service is running on: http://localhost:${port}`); diff --git a/apps/research-service/src/app.module.ts b/apps/research-service/src/app.module.ts index 04074eb..ba44e65 100644 --- a/apps/research-service/src/app.module.ts +++ b/apps/research-service/src/app.module.ts @@ -3,10 +3,11 @@ import { DatabaseModule } from '@terramatch-microservices/database'; import { CommonModule } from '@terramatch-microservices/common'; import { HealthModule } from './health/health.module'; import { SitePolygonsController } from './site-polygons/site-polygons.controller'; +import { SitePolygonsService } from './site-polygons/site-polygons.service'; @Module({ imports: [DatabaseModule, CommonModule, HealthModule], controllers: [SitePolygonsController], - providers: [], + providers: [SitePolygonsService], }) export class AppModule {} diff --git a/apps/research-service/src/main.ts b/apps/research-service/src/main.ts index 455ac07..971a31e 100644 --- a/apps/research-service/src/main.ts +++ b/apps/research-service/src/main.ts @@ -1,28 +1,26 @@ -import { Logger, ValidationPipe } from '@nestjs/common'; -import { NestFactory } from '@nestjs/core'; +import { Logger, ValidationPipe } from "@nestjs/common"; +import { NestFactory } from "@nestjs/core"; -import { AppModule } from './app.module'; -import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger'; -import { TMLogService } from '@terramatch-microservices/common/util/tm-log.service'; +import { AppModule } from "./app.module"; +import { DocumentBuilder, SwaggerModule } from "@nestjs/swagger"; +import { TMLogService } from "@terramatch-microservices/common/util/tm-log.service"; async function bootstrap() { const app = await NestFactory.create(AppModule); const config = new DocumentBuilder() - .setTitle('TerraMatch Research Service') - .setDescription('APIs related to needs for the data research team.') - .setVersion('1.0') - .addTag('research-service') + .setTitle("TerraMatch Research Service") + .setDescription("APIs related to needs for the data research team.") + .setVersion("1.0") + .addTag("research-service") .build(); const document = SwaggerModule.createDocument(app, config); - SwaggerModule.setup('research-service/documentation/api', app, document); + SwaggerModule.setup("research-service/documentation/api", app, document); - app.useGlobalPipes(new ValidationPipe()); + app.useGlobalPipes(new ValidationPipe({ transform: true, transformOptions: { enableImplicitConversion: true } })); app.useLogger(app.get(TMLogService)); - const port = process.env.NODE_ENV === 'production' - ? 80 - : process.env.RESEARCH_SERVICE_PROXY_PORT ?? 4030; + const port = process.env.NODE_ENV === "production" ? 80 : process.env.RESEARCH_SERVICE_PROXY_PORT ?? 4030; await app.listen(port); Logger.log(`TerraMatch Research Service is running on: http://localhost:${port}`); diff --git a/apps/research-service/src/site-polygons/dto/site-polygon-query.dto.ts b/apps/research-service/src/site-polygons/dto/site-polygon-query.dto.ts index db6c42e..8f9b746 100644 --- a/apps/research-service/src/site-polygons/dto/site-polygon-query.dto.ts +++ b/apps/research-service/src/site-polygons/dto/site-polygon-query.dto.ts @@ -1,64 +1,72 @@ -import { ApiProperty } from '@nestjs/swagger'; -import { POLYGON_STATUSES, PolygonStatus } from './site-polygon.dto'; -import { INDICATOR_SLUGS, IndicatorSlug } from './indicators.dto'; +import { ApiProperty } from "@nestjs/swagger"; +import { POLYGON_STATUSES, PolygonStatus } from "./site-polygon.dto"; +import { INDICATOR_SLUGS, IndicatorSlug } from "./indicators.dto"; +import { IsArray, IsDate, IsInt, IsOptional, ValidateNested } from "class-validator"; + +class Page { + @ApiProperty({ + name: "page[size]", + description: "The size of page being requested", + minimum: 1, + maximum: 100, + default: 100 + }) + @IsOptional() + @IsInt() + size?: number; + + @ApiProperty({ + name: "page[after]", + description: + "The last record before the page being requested. The value is a polygon UUID. If not provided, the first page is returned." + }) + @IsOptional() + after?: string; +} export class SitePolygonQueryDto { @ApiProperty({ enum: POLYGON_STATUSES, - name: 'polygonStatus[]', - required: false, + name: "polygonStatus[]", isArray: true, - description: 'Filter results by polygon status', + description: "Filter results by polygon status" }) + @IsOptional() + @IsArray() polygonStatus?: PolygonStatus[]; @ApiProperty({ - name: 'projectId[]', - required: false, + name: "projectId[]", isArray: true, - description: 'Filter results by project UUID(s)', + description: "Filter results by project UUID(s)" }) + @IsOptional() + @IsArray() projectId?: string[]; @ApiProperty({ enum: INDICATOR_SLUGS, - name: 'missingIndicator[]', - required: false, + name: "missingIndicator[]", isArray: true, - description: - 'Filter results by polygons that are missing at least one of the indicators listed', + description: "Filter results by polygons that are missing at least one of the indicators listed" }) + @IsOptional() + @IsArray() missingIndicator?: IndicatorSlug[]; - @ApiProperty({ - required: false, - description: - 'Filter results by polygons that have been modified since the date provided', - }) + @ApiProperty({ description: "Filter results by polygons that have been modified since the date provided" }) + @IsOptional() + @IsDate() lastModifiedDate?: Date; @ApiProperty({ - required: false, - description: - 'Filter results by polygons that are within the boundary of the polygon referenced by this UUID', + description: "Filter results by polygons that are within the boundary of the polygon referenced by this UUID" }) + @IsOptional() boundaryPolygon?: string; - @ApiProperty({ - required: false, - name: 'page[size]', - description: 'The size of page being requested', - minimum: 1, - maximum: 100, - default: 100, - }) - pageSize?: number; - - @ApiProperty({ - required: false, - name: 'page[after]', - description: - 'The last record before the page being requested. The value is a polygon UUID. If not provided, the first page is returned.', - }) - pageAfterCursor?: string; + @ApiProperty({ name: "page", description: "Pagination information" }) + @ValidateNested() + @IsOptional() + page?: Page; } diff --git a/apps/research-service/src/site-polygons/site-polygons.controller.ts b/apps/research-service/src/site-polygons/site-polygons.controller.ts index da51b6d..f065d68 100644 --- a/apps/research-service/src/site-polygons/site-polygons.controller.ts +++ b/apps/research-service/src/site-polygons/site-polygons.controller.ts @@ -1,28 +1,33 @@ import { - BadRequestException, Body, + BadRequestException, + Body, Controller, Get, - NotImplementedException, Patch, + NotImplementedException, + Patch, Query, UnauthorizedException -} from '@nestjs/common'; -import { JsonApiDocument } from '@terramatch-microservices/common/util'; -import { ApiExtraModels, ApiOkResponse, ApiOperation } from '@nestjs/swagger'; -import { ApiException } from '@nanogiants/nestjs-swagger-api-exception-decorator'; -import { JsonApiResponse } from '@terramatch-microservices/common/decorators'; -import { SitePolygonDto } from './dto/site-polygon.dto'; -import { SitePolygonQueryDto } from './dto/site-polygon-query.dto'; +} from "@nestjs/common"; +import { buildJsonApi, JsonApiDocument } from "@terramatch-microservices/common/util"; +import { ApiExtraModels, ApiOkResponse, ApiOperation } from "@nestjs/swagger"; +import { ApiException } from "@nanogiants/nestjs-swagger-api-exception-decorator"; +import { JsonApiResponse } from "@terramatch-microservices/common/decorators"; +import { SitePolygonDto } from "./dto/site-polygon.dto"; +import { SitePolygonQueryDto } from "./dto/site-polygon-query.dto"; import { IndicatorFieldMonitoringDto, IndicatorHectaresDto, IndicatorMsuCarbonDto, IndicatorTreeCountDto, IndicatorTreeCoverDto, - IndicatorTreeCoverLossDto, -} from './dto/indicators.dto'; -import { SitePolygonBulkUpdateBodyDto } from './dto/site-polygon-update.dto'; + IndicatorTreeCoverLossDto +} from "./dto/indicators.dto"; +import { SitePolygonBulkUpdateBodyDto } from "./dto/site-polygon-update.dto"; +import { SitePolygonsService } from "./site-polygons.service"; + +const DEFAULT_PAGE_SIZE = 100 as const; -@Controller('research/v3/sitePolygons') +@Controller("research/v3/sitePolygons") @ApiExtraModels( IndicatorTreeCoverLossDto, IndicatorHectaresDto, @@ -32,31 +37,36 @@ import { SitePolygonBulkUpdateBodyDto } from './dto/site-polygon-update.dto'; IndicatorMsuCarbonDto ) export class SitePolygonsController { + constructor(private readonly sitePolygonService: SitePolygonsService) {} + @Get() - @ApiOperation({ operationId: 'sitePolygonsIndex', summary: 'Get all site polygons' }) + @ApiOperation({ operationId: "sitePolygonsIndex", summary: "Get all site polygons" }) @JsonApiResponse({ data: { type: SitePolygonDto }, hasMany: true, pagination: true }) - @ApiException(() => UnauthorizedException, { description: 'Authentication failed.' }) - @ApiException(() => BadRequestException, { description: 'Pagination values are invalid.' }) - async findMany( - @Query() query?: SitePolygonQueryDto - ): Promise { - throw new NotImplementedException(); + @ApiException(() => UnauthorizedException, { description: "Authentication failed." }) + @ApiException(() => BadRequestException, { description: "Pagination values are invalid." }) + async findMany(@Query() query?: SitePolygonQueryDto): Promise { + const { size: pageSize = DEFAULT_PAGE_SIZE, after: pageAfter } = query.page ?? {}; + if (pageSize > DEFAULT_PAGE_SIZE || pageSize < 1) { + throw new BadRequestException("Page size is invalid"); + } + + const builder = this.sitePolygonService.buildQuery(pageSize, pageAfter); + + const document = buildJsonApi(); + return document.serialize(); } @Patch() @ApiOperation({ - operationId: 'bulkUpdateSitePolygons', - summary: 'Update indicators for site polygons', - description: - `If an indicator is provided that already exists, it will be updated with the value in the + operationId: "bulkUpdateSitePolygons", + summary: "Update indicators for site polygons", + description: `If an indicator is provided that already exists, it will be updated with the value in the payload. If a new indicator is provided, it will be created in the DB. Indicators are keyed off of the combination of site polygon ID, indicatorSlug, and yearOfAnalysis.` }) @ApiOkResponse() - @ApiException(() => UnauthorizedException, { description: 'Authentication failed.' }) - async bulkUpdate( - @Body() updatePayload: SitePolygonBulkUpdateBodyDto - ): Promise { + @ApiException(() => UnauthorizedException, { description: "Authentication failed." }) + async bulkUpdate(@Body() updatePayload: SitePolygonBulkUpdateBodyDto): Promise { throw new NotImplementedException(); } } diff --git a/apps/research-service/src/site-polygons/site-polygons.service.ts b/apps/research-service/src/site-polygons/site-polygons.service.ts new file mode 100644 index 0000000..b93fa74 --- /dev/null +++ b/apps/research-service/src/site-polygons/site-polygons.service.ts @@ -0,0 +1,12 @@ +import { Injectable } from '@nestjs/common'; + +class SitePolygonQueryBuilder { + constructor(private readonly pageSize: number, private readonly pageAfter?: string) {} +} + +@Injectable() +export class SitePolygonsService { + buildQuery(pageSize: number, pageAfter?: string): SitePolygonQueryBuilder { + return new SitePolygonQueryBuilder(pageSize, pageAfter); + } +} diff --git a/apps/user-service/src/main.ts b/apps/user-service/src/main.ts index 7fc4a68..019b07c 100644 --- a/apps/user-service/src/main.ts +++ b/apps/user-service/src/main.ts @@ -1,33 +1,31 @@ -import { Logger, ValidationPipe } from '@nestjs/common'; -import { NestFactory } from '@nestjs/core'; +import { Logger, ValidationPipe } from "@nestjs/common"; +import { NestFactory } from "@nestjs/core"; -import { AppModule } from './app.module'; -import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger'; -import { TMLogService } from '@terramatch-microservices/common/util/tm-log.service'; +import { AppModule } from "./app.module"; +import { DocumentBuilder, SwaggerModule } from "@nestjs/swagger"; +import { TMLogService } from "@terramatch-microservices/common/util/tm-log.service"; async function bootstrap() { const app = await NestFactory.create(AppModule); - if (process.env.NODE_ENV === 'development') { + if (process.env.NODE_ENV === "development") { // CORS is handled by the Api Gateway in AWS app.enableCors(); } const config = new DocumentBuilder() - .setTitle('TerraMatch User Service') - .setDescription('APIs related to login, users and organisations.') - .setVersion('1.0') - .addTag('user-service') + .setTitle("TerraMatch User Service") + .setDescription("APIs related to login, users and organisations.") + .setVersion("1.0") + .addTag("user-service") .build(); const document = SwaggerModule.createDocument(app, config); - SwaggerModule.setup('user-service/documentation/api', app, document); + SwaggerModule.setup("user-service/documentation/api", app, document); - app.useGlobalPipes(new ValidationPipe()); + app.useGlobalPipes(new ValidationPipe({ transform: true, transformOptions: { enableImplicitConversion: true } })); app.useLogger(app.get(TMLogService)); - const port = process.env.NODE_ENV === 'production' - ? 80 - : process.env.USER_SERVICE_PORT ?? 4010; + const port = process.env.NODE_ENV === "production" ? 80 : process.env.USER_SERVICE_PORT ?? 4010; await app.listen(port); Logger.log(`TerraMatch User Service is running on: http://localhost:${port}`); From 22a940a7f2ae5d3435d2e4cbc0750d0a8ab9d413 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Wed, 6 Nov 2024 11:59:03 -0800 Subject: [PATCH 13/41] [TM-1451] Build out database schema for site polygons and associated tables. --- .../dto/site-polygon-query.dto.ts | 2 +- .../src/site-polygons/dto/site-polygon.dto.ts | 94 +++++++------ .../site-polygons/site-polygons.controller.ts | 5 +- .../site-polygons/site-polygons.service.ts | 32 ++++- libs/common/src/lib/util/json-api-builder.ts | 46 +++--- libs/database/src/index.ts | 2 +- libs/database/src/lib/constants/index.ts | 1 + .../src/lib/constants/polygon-status.ts | 2 + libs/database/src/lib/entities/index.ts | 26 ++-- .../src/lib/entities/point-geometry.entity.ts | 34 +++++ .../lib/entities/polygon-geometry.entity.ts | 25 ++++ .../src/lib/entities/project.entity.ts | 8 +- .../src/lib/entities/site-polygon.entity.ts | 131 ++++++++++++++++++ libs/database/src/lib/entities/site.entity.ts | 15 ++ package-lock.json | 10 ++ package.json | 2 + 16 files changed, 347 insertions(+), 88 deletions(-) create mode 100644 libs/database/src/lib/constants/index.ts create mode 100644 libs/database/src/lib/constants/polygon-status.ts create mode 100644 libs/database/src/lib/entities/point-geometry.entity.ts create mode 100644 libs/database/src/lib/entities/polygon-geometry.entity.ts create mode 100644 libs/database/src/lib/entities/site-polygon.entity.ts create mode 100644 libs/database/src/lib/entities/site.entity.ts diff --git a/apps/research-service/src/site-polygons/dto/site-polygon-query.dto.ts b/apps/research-service/src/site-polygons/dto/site-polygon-query.dto.ts index 8f9b746..fa6e1c8 100644 --- a/apps/research-service/src/site-polygons/dto/site-polygon-query.dto.ts +++ b/apps/research-service/src/site-polygons/dto/site-polygon-query.dto.ts @@ -1,7 +1,7 @@ import { ApiProperty } from "@nestjs/swagger"; -import { POLYGON_STATUSES, PolygonStatus } from "./site-polygon.dto"; import { INDICATOR_SLUGS, IndicatorSlug } from "./indicators.dto"; import { IsArray, IsDate, IsInt, IsOptional, ValidateNested } from "class-validator"; +import { POLYGON_STATUSES, PolygonStatus } from "@terramatch-microservices/database/constants"; class Page { @ApiProperty({ diff --git a/apps/research-service/src/site-polygons/dto/site-polygon.dto.ts b/apps/research-service/src/site-polygons/dto/site-polygon.dto.ts index ed97766..4870341 100644 --- a/apps/research-service/src/site-polygons/dto/site-polygon.dto.ts +++ b/apps/research-service/src/site-polygons/dto/site-polygon.dto.ts @@ -1,15 +1,19 @@ -import { JsonApiAttributes } from '@terramatch-microservices/common/dto/json-api-attributes'; -import { JsonApiDto } from '@terramatch-microservices/common/decorators'; -import { ApiProperty } from '@nestjs/swagger'; +import { JsonApiAttributes } from "@terramatch-microservices/common/dto/json-api-attributes"; +import { JsonApiDto } from "@terramatch-microservices/common/decorators"; +import { ApiProperty } from "@nestjs/swagger"; import { IndicatorFieldMonitoringDto, - IndicatorHectaresDto, IndicatorMsuCarbonDto, - IndicatorTreeCountDto, IndicatorTreeCoverDto, + IndicatorHectaresDto, + IndicatorMsuCarbonDto, + IndicatorTreeCountDto, + IndicatorTreeCoverDto, IndicatorTreeCoverLossDto -} from './indicators.dto'; +} from "./indicators.dto"; +import { POLYGON_STATUSES, PolygonStatus } from "@terramatch-microservices/database/constants"; +import { SitePolygon } from "@terramatch-microservices/database/entities"; class TreeSpecies { - @ApiProperty({ example: 'Acacia binervia' }) + @ApiProperty({ example: "Acacia binervia" }) name: string; @ApiProperty({ example: 15000 }) @@ -26,21 +30,31 @@ class ReportingPeriod { @ApiProperty({ type: () => TreeSpecies, isArray: true, - description: 'The tree species reported as planted during this reporting period' + description: "The tree species reported as planted during this reporting period" }) treeSpecies: TreeSpecies[]; } -export const POLYGON_STATUSES = [ - 'draft', - 'submitted', - 'needs-more-information', - 'approved' -]; -export type PolygonStatus = (typeof POLYGON_STATUSES)[number]; - -@JsonApiDto({ type: 'sitePolygons' }) +@JsonApiDto({ type: "sitePolygons" }) export class SitePolygonDto extends JsonApiAttributes { + constructor(sitePolygon: SitePolygon) { + super({ + name: sitePolygon.polyName, + status: sitePolygon.status, + siteId: sitePolygon.siteUuid, + plantStart: sitePolygon.plantStart, + plantEnd: sitePolygon.plantEnd, + practice: sitePolygon.practice, + targetSys: sitePolygon.targetSys, + distr: sitePolygon.distr, + numTrees: sitePolygon.numTrees, + calcArea: sitePolygon.calcArea, + indicators: [], + establishmentTreeSpecies: [], + reportingPeriods: [] + }); + } + @ApiProperty() name: string; @@ -51,60 +65,60 @@ export class SitePolygonDto extends JsonApiAttributes { siteId: string; @ApiProperty() - plantStart: Date; + plantStart: Date | null; @ApiProperty() - plantEnd: Date; + plantEnd: Date | null; @ApiProperty() - practice: string; + practice: string | null; @ApiProperty() - targetSys: string; + targetSys: string | null; @ApiProperty() - distr: string; + distr: string | null; @ApiProperty() - numTrees: number; + numTrees: number | null; @ApiProperty() - calcArea: number; + calcArea: number | null; @ApiProperty({ - type: 'array', + type: "array", items: { oneOf: [ - { $ref: '#/components/schemas/IndicatorTreeCoverLossDto' }, - { $ref: '#/components/schemas/IndicatorHectaresDto' }, - { $ref: '#/components/schemas/IndicatorTreeCountDto' }, - { $ref: '#/components/schemas/IndicatorTreeCoverDto' }, - { $ref: '#/components/schemas/IndicatorFieldMonitoringDto' }, - { $ref: '#/components/schemas/IndicatorMsuCarbonDto' }, + { $ref: "#/components/schemas/IndicatorTreeCoverLossDto" }, + { $ref: "#/components/schemas/IndicatorHectaresDto" }, + { $ref: "#/components/schemas/IndicatorTreeCountDto" }, + { $ref: "#/components/schemas/IndicatorTreeCoverDto" }, + { $ref: "#/components/schemas/IndicatorFieldMonitoringDto" }, + { $ref: "#/components/schemas/IndicatorMsuCarbonDto" } ] }, - description: 'All indicators currently recorded for this site polygon' + description: "All indicators currently recorded for this site polygon" }) indicators: ( - IndicatorTreeCoverLossDto | - IndicatorHectaresDto | - IndicatorTreeCountDto | - IndicatorTreeCoverDto | - IndicatorFieldMonitoringDto | - IndicatorMsuCarbonDto + | IndicatorTreeCoverLossDto + | IndicatorHectaresDto + | IndicatorTreeCountDto + | IndicatorTreeCoverDto + | IndicatorFieldMonitoringDto + | IndicatorMsuCarbonDto )[]; @ApiProperty({ type: () => TreeSpecies, isArray: true, - description: 'The tree species associated with the establishment of the site that this polygon relates to.' + description: "The tree species associated with the establishment of the site that this polygon relates to." }) establishmentTreeSpecies: TreeSpecies[]; @ApiProperty({ type: () => ReportingPeriod, isArray: true, - description: 'Access to reported trees planted for each approved report on this site.' + description: "Access to reported trees planted for each approved report on this site." }) reportingPeriods: ReportingPeriod[]; } diff --git a/apps/research-service/src/site-polygons/site-polygons.controller.ts b/apps/research-service/src/site-polygons/site-polygons.controller.ts index f065d68..3cdaea4 100644 --- a/apps/research-service/src/site-polygons/site-polygons.controller.ts +++ b/apps/research-service/src/site-polygons/site-polygons.controller.ts @@ -50,9 +50,12 @@ export class SitePolygonsController { throw new BadRequestException("Page size is invalid"); } - const builder = this.sitePolygonService.buildQuery(pageSize, pageAfter); + const builder = await this.sitePolygonService.buildQuery(pageSize, pageAfter); const document = buildJsonApi(); + for (const sitePolygon of await builder.execute()) { + document.addData(sitePolygon.uuid, new SitePolygonDto(sitePolygon)); + } return document.serialize(); } diff --git a/apps/research-service/src/site-polygons/site-polygons.service.ts b/apps/research-service/src/site-polygons/site-polygons.service.ts index b93fa74..c52f55a 100644 --- a/apps/research-service/src/site-polygons/site-polygons.service.ts +++ b/apps/research-service/src/site-polygons/site-polygons.service.ts @@ -1,12 +1,36 @@ -import { Injectable } from '@nestjs/common'; +import { BadRequestException, Injectable } from "@nestjs/common"; +import { SitePolygon } from "@terramatch-microservices/database/entities"; +import { Attributes, FindOptions, Op, WhereOptions } from "sequelize"; class SitePolygonQueryBuilder { - constructor(private readonly pageSize: number, private readonly pageAfter?: string) {} + private findOptions: FindOptions> = {}; + + constructor(pageSize: number) { + this.findOptions.limit = pageSize; + } + + async pageAfter(pageAfter: string) { + const sitePolygon = await SitePolygon.findOne({ where: { uuid: pageAfter }, attributes: ["id"] }); + if (sitePolygon == null) throw new BadRequestException("pageAfter polygon not found"); + this.where({ id: { [Op.gt]: sitePolygon.id } }); + return this; + } + + async execute(): Promise { + return await SitePolygon.findAll(this.findOptions); + } + + private where(options: WhereOptions) { + if (this.findOptions.where == null) this.findOptions.where = {}; + Object.assign(this.findOptions.where, options); + } } @Injectable() export class SitePolygonsService { - buildQuery(pageSize: number, pageAfter?: string): SitePolygonQueryBuilder { - return new SitePolygonQueryBuilder(pageSize, pageAfter); + async buildQuery(pageSize: number, pageAfter?: string) { + const builder = new SitePolygonQueryBuilder(pageSize); + if (pageAfter != null) await builder.pageAfter(pageAfter); + return builder; } } diff --git a/libs/common/src/lib/util/json-api-builder.ts b/libs/common/src/lib/util/json-api-builder.ts index 313f1d6..4f607c3 100644 --- a/libs/common/src/lib/util/json-api-builder.ts +++ b/libs/common/src/lib/util/json-api-builder.ts @@ -1,33 +1,33 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ -import { DTO_TYPE_METADATA } from '../decorators/json-api-dto.decorator'; -import { InternalServerErrorException } from '@nestjs/common'; +import { DTO_TYPE_METADATA } from "../decorators/json-api-dto.decorator"; +import { InternalServerErrorException } from "@nestjs/common"; type AttributeValue = string | number | boolean; type Attributes = { - [key: string]: AttributeValue | Attributes -} + [key: string]: AttributeValue | Attributes; +}; export type Relationship = { type: string; id: string; meta?: Attributes; -} +}; export type Relationships = { - [key: string]: { data: Relationship | Relationship[] } -} + [key: string]: { data: Relationship | Relationship[] }; +}; export type Resource = { type: string; id: string; attributes: Attributes; relationships?: Relationships; -} +}; export type JsonApiDocument = { data: Resource | Resource[]; included?: Resource | Resource[]; -} +}; export class ResourceBuilder { type: string; @@ -36,7 +36,7 @@ export class ResourceBuilder { constructor(public id: string, public attributes: Attributes, private documentBuilder: DocumentBuilder) { this.type = Reflect.getMetadata(DTO_TYPE_METADATA, attributes.constructor); - if (this.type == null && process.env['NODE_ENV'] !== 'production') { + if (this.type == null && process.env["NODE_ENV"] !== "production") { throw new InternalServerErrorException( `Attribute types are required to use the @JsonApiDto decorator [${this.constructor.name}]` ); @@ -47,7 +47,7 @@ export class ResourceBuilder { return this.documentBuilder; } - relateTo(label: string, resource: { id: string, type: string }, meta?: Attributes): ResourceBuilder { + relateTo(label: string, resource: { id: string; type: string }, meta?: Attributes): ResourceBuilder { if (this.relationships == null) this.relationships = {}; // This method signature was created so that another resource builder could be passed in for the @@ -56,7 +56,7 @@ export class ResourceBuilder { const { id, type } = resource; const relationship = { id, type, meta }; if (this.relationships[label] == null) { - this.relationships[label] = { data: relationship } + this.relationships[label] = { data: relationship }; } else if (Array.isArray(this.relationships[label].data)) { this.relationships[label].data.push(relationship); } else { @@ -70,7 +70,7 @@ export class ResourceBuilder { const resource = { type: this.type, id: this.id, - attributes: this.attributes, + attributes: this.attributes } as Resource; if (this.relationships != null) { @@ -92,7 +92,9 @@ class DocumentBuilder { const matchesType = this.data.length == 0 || this.data[0].type === builder.type; if (!matchesType) { - throw new ApiBuilderException(`This resource does not match the data type [${builder.type}, ${this.data[0].type}]`) + throw new ApiBuilderException( + `This resource does not match the data type [${builder.type}, ${this.data[0].type}]` + ); } const collision = this.data.find(({ id: existingId }) => existingId === id); @@ -107,9 +109,7 @@ class DocumentBuilder { addIncluded(id: string, attributes: any): ResourceBuilder { const builder = new ResourceBuilder(id, attributes, this); - const collision = this.included.find( - ({ type, id: existingId }) => existingId === id && type === builder.type - ); + const collision = this.included.find(({ type, id: existingId }) => existingId === id && type === builder.type); if (collision != null) { throw new ApiBuilderException(`This resource is already included [${id}, ${builder.type}]`); } @@ -119,20 +119,14 @@ class DocumentBuilder { } serialize(): JsonApiDocument { - if (this.data.length === 0) { - throw new ApiBuilderException('Cannot build a document with no data!'); - } - const doc: JsonApiDocument = { // Data can either be a single object or an array - data: this.data.length === 1 - ? this.data[0].serialize() - : this.data.map((resource) => resource.serialize()) - } + data: this.data.length === 1 ? this.data[0].serialize() : this.data.map(resource => resource.serialize()) + }; if (this.included.length > 0) { // Included is always an array - doc.included = this.included.map((resource) => resource.serialize()); + doc.included = this.included.map(resource => resource.serialize()); } return doc; diff --git a/libs/database/src/index.ts b/libs/database/src/index.ts index ae557f8..aa42a31 100644 --- a/libs/database/src/index.ts +++ b/libs/database/src/index.ts @@ -1 +1 @@ -export * from './lib/database.module'; +export * from "./lib/database.module"; diff --git a/libs/database/src/lib/constants/index.ts b/libs/database/src/lib/constants/index.ts new file mode 100644 index 0000000..a1212e1 --- /dev/null +++ b/libs/database/src/lib/constants/index.ts @@ -0,0 +1 @@ +export * from "./polygon-status"; diff --git a/libs/database/src/lib/constants/polygon-status.ts b/libs/database/src/lib/constants/polygon-status.ts new file mode 100644 index 0000000..9b9668f --- /dev/null +++ b/libs/database/src/lib/constants/polygon-status.ts @@ -0,0 +1,2 @@ +export const POLYGON_STATUSES = ["draft", "submitted", "needs-more-information", "approved"]; +export type PolygonStatus = (typeof POLYGON_STATUSES)[number]; diff --git a/libs/database/src/lib/entities/index.ts b/libs/database/src/lib/entities/index.ts index 0967a28..704f06b 100644 --- a/libs/database/src/lib/entities/index.ts +++ b/libs/database/src/lib/entities/index.ts @@ -1,11 +1,15 @@ -export * from './delayed-job.entity'; -export * from './framework.entity'; -export * from './framework-user.entity'; -export * from './model-has-role.entity' -export * from './organisation.entity'; -export * from './organisation-user.entity'; -export * from './permission.entity'; -export * from './project.entity'; -export * from './project-user.entity' -export * from './role.entity'; -export * from './user.entity'; +export * from "./delayed-job.entity"; +export * from "./framework.entity"; +export * from "./framework-user.entity"; +export * from "./model-has-role.entity"; +export * from "./organisation.entity"; +export * from "./organisation-user.entity"; +export * from "./permission.entity"; +export * from "./point-geometry.entity"; +export * from "./polygon-geometry.entity"; +export * from "./project.entity"; +export * from "./project-user.entity"; +export * from "./role.entity"; +export * from "./site.entity"; +export * from "./site-polygon.entity"; +export * from "./user.entity"; diff --git a/libs/database/src/lib/entities/point-geometry.entity.ts b/libs/database/src/lib/entities/point-geometry.entity.ts new file mode 100644 index 0000000..6d37d3d --- /dev/null +++ b/libs/database/src/lib/entities/point-geometry.entity.ts @@ -0,0 +1,34 @@ +import { AllowNull, AutoIncrement, Column, ForeignKey, Index, Model, PrimaryKey, Table } from "sequelize-typescript"; +import { BIGINT, DECIMAL, GEOMETRY, UUID } from "sequelize"; +import { Point } from "geojson"; +import { User } from "./user.entity"; + +@Table({ tableName: "point_geometry", underscored: true, paranoid: true }) +export class PointGeometry extends Model { + @PrimaryKey + @AutoIncrement + @Column(BIGINT.UNSIGNED) + override id: number; + + @Index + @Column(UUID) + uuid: string; + + @AllowNull + @Column({ type: GEOMETRY, field: "geom" }) + point: Point; + + @AllowNull + @Column({ type: DECIMAL(15, 2), field: "est_area" }) + estimatedArea: number; + + @ForeignKey(() => User) + @AllowNull + @Column(BIGINT.UNSIGNED) + createdBy: number | null; + + @ForeignKey(() => User) + @AllowNull + @Column(BIGINT.UNSIGNED) + lastModifiedBy: number | null; +} diff --git a/libs/database/src/lib/entities/polygon-geometry.entity.ts b/libs/database/src/lib/entities/polygon-geometry.entity.ts new file mode 100644 index 0000000..27b3ea4 --- /dev/null +++ b/libs/database/src/lib/entities/polygon-geometry.entity.ts @@ -0,0 +1,25 @@ +import { AllowNull, AutoIncrement, Column, ForeignKey, Index, Model, PrimaryKey, Table } from "sequelize-typescript"; +import { BIGINT, GEOMETRY, UUID } from "sequelize"; +import { Polygon } from "geojson"; +import { User } from "./user.entity"; + +@Table({ tableName: "polygon_geometry", underscored: true, paranoid: true }) +export class PolygonGeometry extends Model { + @PrimaryKey + @AutoIncrement + @Column(BIGINT.UNSIGNED) + override id: number; + + @Index + @Column(UUID) + uuid: string; + + @AllowNull + @Column({ type: GEOMETRY, field: "geom" }) + polygon: Polygon; + + @ForeignKey(() => User) + @AllowNull + @Column(BIGINT.UNSIGNED) + createdBy: number | null; +} diff --git a/libs/database/src/lib/entities/project.entity.ts b/libs/database/src/lib/entities/project.entity.ts index abe8dd6..9371dca 100644 --- a/libs/database/src/lib/entities/project.entity.ts +++ b/libs/database/src/lib/entities/project.entity.ts @@ -1,8 +1,8 @@ -import { AutoIncrement, Column, Model, PrimaryKey, Table } from 'sequelize-typescript'; -import { BIGINT, STRING } from 'sequelize'; +import { AutoIncrement, Column, Model, PrimaryKey, Table } from "sequelize-typescript"; +import { BIGINT, STRING } from "sequelize"; -// A quick stub to get The information needed for users/me -@Table({ tableName: 'v2_projects', underscored: true }) +// A quick stub to get the information needed for users/me +@Table({ tableName: "v2_projects", underscored: true, paranoid: true }) export class Project extends Model { @PrimaryKey @AutoIncrement diff --git a/libs/database/src/lib/entities/site-polygon.entity.ts b/libs/database/src/lib/entities/site-polygon.entity.ts new file mode 100644 index 0000000..9fb3d76 --- /dev/null +++ b/libs/database/src/lib/entities/site-polygon.entity.ts @@ -0,0 +1,131 @@ +import { + AllowNull, + AutoIncrement, + BelongsTo, + Column, + Default, + ForeignKey, + Index, + Model, + PrimaryKey, + Table +} from "sequelize-typescript"; +import { BIGINT, BOOLEAN, DATE, DOUBLE, INTEGER, STRING, UUID } from "sequelize"; +import { Site } from "./site.entity"; +import { PointGeometry } from "./point-geometry.entity"; +import { PolygonGeometry } from "./polygon-geometry.entity"; +import { User } from "./user.entity"; +import { POLYGON_STATUSES, PolygonStatus } from "../constants"; + +@Table({ tableName: "site_polygon", underscored: true, paranoid: true }) +export class SitePolygon extends Model { + @PrimaryKey + @AutoIncrement + @Column(BIGINT.UNSIGNED) + override id: number; + + @Index + @Column(UUID) + uuid: string; + + @Column(UUID) + primaryUuid: string; + + // This column got called site_id in the PHP codebase, which is misleading because it's a UUID + @AllowNull + @Column({ type: UUID, field: "site_id" }) + siteUuid: string; + + @BelongsTo(() => Site, { foreignKey: "siteUuid", targetKey: "uuid" }) + site: Site | null; + + async loadSite() { + if (this.site == null && this.siteUuid != null) { + this.site = await this.$get("site"); + } + return this.site; + } + + // This column got called point_id in the PHP codebase, which is misleading because it's a UUID + @AllowNull + @Column({ type: UUID, field: "point_id" }) + pointUuid: string; + + @BelongsTo(() => PointGeometry, { foreignKey: "pointUuid", targetKey: "uuid" }) + point: PointGeometry | null; + + async loadPoint() { + if (this.point == null && this.pointUuid != null) { + this.point = await this.$get("point"); + } + return this.point; + } + + // This column got called poly_id in the PHP codebase, which is misleading because it's a UUID + @AllowNull + @Column({ type: UUID, field: "poly_id" }) + polygonUuid: string; + + @BelongsTo(() => PolygonGeometry, { foreignKey: "polygonUuid", targetKey: "uuid" }) + polygon: PolygonGeometry | null; + + async loadPolygon() { + if (this.polygon == null && this.polygonUuid != null) { + this.polygon = await this.$get("polygon"); + } + return this.polygon; + } + + @AllowNull + @Column(STRING) + polyName: string | null; + + @AllowNull + @Column({ type: DATE, field: "plantstart" }) + plantStart: Date | null; + + @AllowNull + @Column({ type: DATE, field: "plantend" }) + plantEnd: Date | null; + + @AllowNull + @Column(STRING) + practice: string | null; + + @AllowNull + @Column(STRING) + targetSys: string | null; + + @AllowNull + @Column(STRING) + distr: string | null; + + @AllowNull + @Column(INTEGER) + numTrees: number | null; + + @AllowNull + @Column(DOUBLE) + calcArea: number | null; + + @AllowNull + @Column({ type: STRING, values: POLYGON_STATUSES }) + status: PolygonStatus | null; + + @AllowNull + @Column(STRING) + source: string | null; + + @ForeignKey(() => User) + @AllowNull + @Column(BIGINT.UNSIGNED) + createdBy: number | null; + + @Default(false) + @Column(BOOLEAN) + isActive: boolean; + + @AllowNull + @Column(STRING) + versionName: string | null; +} diff --git a/libs/database/src/lib/entities/site.entity.ts b/libs/database/src/lib/entities/site.entity.ts new file mode 100644 index 0000000..0b96383 --- /dev/null +++ b/libs/database/src/lib/entities/site.entity.ts @@ -0,0 +1,15 @@ +import { AutoIncrement, Column, Index, Model, PrimaryKey, Table } from "sequelize-typescript"; +import { BIGINT, UUID } from "sequelize"; + +// A quick stub for the research endpoints +@Table({ tableName: "v2_sites", underscored: true, paranoid: true }) +export class Site extends Model { + @PrimaryKey + @AutoIncrement + @Column(BIGINT.UNSIGNED) + override id: number; + + @Index + @Column(UUID) + uuid: string; +} diff --git a/package-lock.json b/package-lock.json index f4b0b90..356e694 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,6 +23,7 @@ "bcryptjs": "^2.4.3", "class-transformer": "^0.5.1", "class-validator": "^0.14.1", + "geojson": "^0.5.0", "lodash": "^4.17.21", "mariadb": "^3.3.2", "mysql2": "^3.11.2", @@ -51,6 +52,7 @@ "@swc/core": "~1.5.7", "@swc/helpers": "~0.5.11", "@types/bcryptjs": "^2.4.6", + "@types/geojson": "^7946.0.14", "@types/jest": "^29.5.12", "@types/lodash": "^4.17.13", "@types/node": "~18.16.9", @@ -8765,6 +8767,14 @@ "node": ">=6.9.0" } }, + "node_modules/geojson": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/geojson/-/geojson-0.5.0.tgz", + "integrity": "sha512-/Bx5lEn+qRF4TfQ5aLu6NH+UKtvIv7Lhc487y/c8BdludrCTpiWf9wyI0RTyqg49MFefIAvFDuEi5Dfd/zgNxQ==", + "engines": { + "node": ">= 0.10" + } + }, "node_modules/get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", diff --git a/package.json b/package.json index c1a4189..a87214c 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "bcryptjs": "^2.4.3", "class-transformer": "^0.5.1", "class-validator": "^0.14.1", + "geojson": "^0.5.0", "lodash": "^4.17.21", "mariadb": "^3.3.2", "mysql2": "^3.11.2", @@ -51,6 +52,7 @@ "@swc/core": "~1.5.7", "@swc/helpers": "~0.5.11", "@types/bcryptjs": "^2.4.6", + "@types/geojson": "^7946.0.14", "@types/jest": "^29.5.12", "@types/lodash": "^4.17.13", "@types/node": "~18.16.9", From 6c542a265612cca849563a496f5fb3b79fc3f956 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Wed, 6 Nov 2024 17:01:46 -0800 Subject: [PATCH 14/41] [TM-1451] Define indicator tables. --- .../src/site-polygons/dto/indicators.dto.ts | 110 +++++++++--------- .../dto/site-polygon-query.dto.ts | 8 +- .../src/site-polygons/dto/site-polygon.dto.ts | 21 ++-- .../site-polygons/site-polygons.controller.ts | 5 +- .../site-polygons/site-polygons.service.ts | 29 ++++- libs/database/src/lib/constants/index.ts | 1 + .../src/lib/constants/polygon-indicators.ts | 15 +++ libs/database/src/lib/entities/index.ts | 6 + ...ndicator-output-field-monitoring.entity.ts | 41 +++++++ .../indicator-output-hectares.entity.ts | 28 +++++ .../indicator-output-msu-carbon.entity.ts | 37 ++++++ .../indicator-output-tree-count.entity.ts | 61 ++++++++++ ...indicator-output-tree-cover-loss.entity.ts | 28 +++++ .../indicator-output-tree-cover.entity.ts | 37 ++++++ .../src/lib/entities/site-polygon.entity.ts | 75 +++++++++++- 15 files changed, 430 insertions(+), 72 deletions(-) create mode 100644 libs/database/src/lib/constants/polygon-indicators.ts create mode 100644 libs/database/src/lib/entities/indicator-output-field-monitoring.entity.ts create mode 100644 libs/database/src/lib/entities/indicator-output-hectares.entity.ts create mode 100644 libs/database/src/lib/entities/indicator-output-msu-carbon.entity.ts create mode 100644 libs/database/src/lib/entities/indicator-output-tree-count.entity.ts create mode 100644 libs/database/src/lib/entities/indicator-output-tree-cover-loss.entity.ts create mode 100644 libs/database/src/lib/entities/indicator-output-tree-cover.entity.ts diff --git a/apps/research-service/src/site-polygons/dto/indicators.dto.ts b/apps/research-service/src/site-polygons/dto/indicators.dto.ts index 6912692..f6be989 100644 --- a/apps/research-service/src/site-polygons/dto/indicators.dto.ts +++ b/apps/research-service/src/site-polygons/dto/indicators.dto.ts @@ -1,133 +1,131 @@ -import { ApiProperty } from '@nestjs/swagger'; - -// Matches the indicators defined on https://gfw.atlassian.net/wiki/spaces/TerraMatch/pages/1469448210/Indicator+Data+Model -export const INDICATORS = { - 1: 'treeCover', - 2: 'treeCoverLoss', - 3: 'treeCoverLossFires', - 4: 'restorationByEcoRegion', - 5: 'restorationByStrategy', - 6: 'restorationByLandUse', - 7: 'treeCount', - 8: 'earlyTreeVerification', - 9: 'fieldMonitoring', - 10: 'msuCarbon' -} as const; -export const INDICATOR_SLUGS = Object.values(INDICATORS); -export type IndicatorSlug = (typeof INDICATOR_SLUGS)[number]; +import { ApiProperty } from "@nestjs/swagger"; +import { INDICATORS } from "@terramatch-microservices/database/constants"; export class IndicatorTreeCoverLossDto { @ApiProperty({ enum: [INDICATORS[2], INDICATORS[3]] }) - indicatorSlug: typeof INDICATORS[2] | typeof INDICATORS[3] + indicatorSlug: (typeof INDICATORS)[2] | (typeof INDICATORS)[3]; - @ApiProperty({ example: '2024' }) + @ApiProperty({ example: "2024" }) yearOfAnalysis: number; @ApiProperty({ - type: 'object', - description: 'Mapping of year of analysis to value.', - example: { 2024: '0.6', 2023: '0.5' } + type: "object", + description: "Mapping of year of analysis to value.", + example: { 2024: "0.6", 2023: "0.5" } }) value: Record; } export class IndicatorHectaresDto { @ApiProperty({ enum: [INDICATORS[4], INDICATORS[5], INDICATORS[6]] }) - indicatorSlug: typeof INDICATORS[4] | typeof INDICATORS[5] | typeof INDICATORS[6]; + indicatorSlug: (typeof INDICATORS)[4] | (typeof INDICATORS)[5] | (typeof INDICATORS)[6]; - @ApiProperty({ example: '2024' }) + @ApiProperty({ example: "2024" }) yearOfAnalysis: number; @ApiProperty({ - type: 'object', - description: 'Mapping of area type (eco region, land use, etc) to hectares', - example: { 'Northern Acacia-Commiphora bushlands and thickets': 0.104 } + type: "object", + description: "Mapping of area type (eco region, land use, etc) to hectares", + example: { "Northern Acacia-Commiphora bushlands and thickets": 0.104 } }) value: Record; } export class IndicatorTreeCountDto { @ApiProperty({ enum: [INDICATORS[7], INDICATORS[8]] }) - indicatorSlug: typeof INDICATORS[7] | typeof INDICATORS[8]; + indicatorSlug: (typeof INDICATORS)[7] | (typeof INDICATORS)[8]; - @ApiProperty({ example: '2024' }) + @ApiProperty({ example: "2024" }) yearOfAnalysis: number; @ApiProperty() - surveyType: string; + surveyType: string | null; @ApiProperty() - surveyId: number; + surveyId: number | null; @ApiProperty() - treeCount: number; + treeCount: number | null; - @ApiProperty({ example: 'types TBD' }) - uncertaintyType: string; + @ApiProperty({ example: "types TBD" }) + uncertaintyType: string | null; @ApiProperty() - imagerySource: string; + imagerySource: string | null; - @ApiProperty({ type: 'url' }) - imageryId: string; + @ApiProperty({ type: "url" }) + imageryId: string | null; @ApiProperty() - projectPhase: string; + projectPhase: string | null; @ApiProperty() - confidence: number; + confidence: number | null; } export class IndicatorTreeCoverDto { @ApiProperty({ enum: [INDICATORS[1]] }) - indicatorSlug: typeof INDICATORS[1]; + indicatorSlug: (typeof INDICATORS)[1]; - @ApiProperty({ example: '2024' }) + @ApiProperty({ example: "2024" }) yearOfAnalysis: number; - @ApiProperty({ example: '2024' }) - projectPhase: string; + @ApiProperty({ example: "2024" }) + projectPhase: string | null; @ApiProperty() - percentCover: number; + percentCover: number | null; @ApiProperty() - plusMinusPercent: number + plusMinusPercent: number | null; } export class IndicatorFieldMonitoringDto { @ApiProperty({ enum: [INDICATORS[9]] }) - indicatorSlug: typeof INDICATORS[9]; + indicatorSlug: (typeof INDICATORS)[9]; - @ApiProperty({ example: '2024' }) + @ApiProperty({ example: "2024" }) yearOfAnalysis: number; @ApiProperty() - treeCount: number; + treeCount: number | null; @ApiProperty() - projectPhase: string; + projectPhase: string | null; @ApiProperty() - species: string; + species: string | null; @ApiProperty() - survivalRate: number; + survivalRate: number | null; } export class IndicatorMsuCarbonDto { @ApiProperty({ enum: [INDICATORS[10]] }) - indicatorSlug: typeof INDICATORS[10]; + indicatorSlug: (typeof INDICATORS)[10]; - @ApiProperty({ example: '2024' }) + @ApiProperty({ example: "2024" }) yearOfAnalysis: number; @ApiProperty() - carbonOutput: number; + carbonOutput: number | null; @ApiProperty() - projectPhase: string; + projectPhase: string | null; @ApiProperty() - confidence: number; + confidence: number | null; } + +export const INDICATOR_DTOS = { + [INDICATORS[1]]: IndicatorTreeCoverDto.prototype, + [INDICATORS[2]]: IndicatorTreeCoverLossDto.prototype, + [INDICATORS[3]]: IndicatorTreeCoverLossDto.prototype, + [INDICATORS[4]]: IndicatorHectaresDto.prototype, + [INDICATORS[5]]: IndicatorHectaresDto.prototype, + [INDICATORS[6]]: IndicatorHectaresDto.prototype, + [INDICATORS[7]]: IndicatorTreeCountDto.prototype, + [INDICATORS[8]]: IndicatorTreeCountDto.prototype, + [INDICATORS[9]]: IndicatorFieldMonitoringDto.prototype, + [INDICATORS[10]]: IndicatorMsuCarbonDto.prototype +}; diff --git a/apps/research-service/src/site-polygons/dto/site-polygon-query.dto.ts b/apps/research-service/src/site-polygons/dto/site-polygon-query.dto.ts index fa6e1c8..0d156b2 100644 --- a/apps/research-service/src/site-polygons/dto/site-polygon-query.dto.ts +++ b/apps/research-service/src/site-polygons/dto/site-polygon-query.dto.ts @@ -1,7 +1,11 @@ import { ApiProperty } from "@nestjs/swagger"; -import { INDICATOR_SLUGS, IndicatorSlug } from "./indicators.dto"; import { IsArray, IsDate, IsInt, IsOptional, ValidateNested } from "class-validator"; -import { POLYGON_STATUSES, PolygonStatus } from "@terramatch-microservices/database/constants"; +import { + INDICATOR_SLUGS, + IndicatorSlug, + POLYGON_STATUSES, + PolygonStatus +} from "@terramatch-microservices/database/constants"; class Page { @ApiProperty({ diff --git a/apps/research-service/src/site-polygons/dto/site-polygon.dto.ts b/apps/research-service/src/site-polygons/dto/site-polygon.dto.ts index 4870341..8e3c553 100644 --- a/apps/research-service/src/site-polygons/dto/site-polygon.dto.ts +++ b/apps/research-service/src/site-polygons/dto/site-polygon.dto.ts @@ -12,6 +12,14 @@ import { import { POLYGON_STATUSES, PolygonStatus } from "@terramatch-microservices/database/constants"; import { SitePolygon } from "@terramatch-microservices/database/entities"; +export type IndicatorDto = + | IndicatorTreeCoverLossDto + | IndicatorHectaresDto + | IndicatorTreeCountDto + | IndicatorTreeCoverDto + | IndicatorFieldMonitoringDto + | IndicatorMsuCarbonDto; + class TreeSpecies { @ApiProperty({ example: "Acacia binervia" }) name: string; @@ -37,7 +45,7 @@ class ReportingPeriod { @JsonApiDto({ type: "sitePolygons" }) export class SitePolygonDto extends JsonApiAttributes { - constructor(sitePolygon: SitePolygon) { + constructor(sitePolygon: SitePolygon, indicators: IndicatorDto[]) { super({ name: sitePolygon.polyName, status: sitePolygon.status, @@ -49,7 +57,7 @@ export class SitePolygonDto extends JsonApiAttributes { distr: sitePolygon.distr, numTrees: sitePolygon.numTrees, calcArea: sitePolygon.calcArea, - indicators: [], + indicators, establishmentTreeSpecies: [], reportingPeriods: [] }); @@ -99,14 +107,7 @@ export class SitePolygonDto extends JsonApiAttributes { }, description: "All indicators currently recorded for this site polygon" }) - indicators: ( - | IndicatorTreeCoverLossDto - | IndicatorHectaresDto - | IndicatorTreeCountDto - | IndicatorTreeCoverDto - | IndicatorFieldMonitoringDto - | IndicatorMsuCarbonDto - )[]; + indicators: IndicatorDto[]; @ApiProperty({ type: () => TreeSpecies, diff --git a/apps/research-service/src/site-polygons/site-polygons.controller.ts b/apps/research-service/src/site-polygons/site-polygons.controller.ts index 3cdaea4..dd3df8c 100644 --- a/apps/research-service/src/site-polygons/site-polygons.controller.ts +++ b/apps/research-service/src/site-polygons/site-polygons.controller.ts @@ -54,7 +54,10 @@ export class SitePolygonsController { const document = buildJsonApi(); for (const sitePolygon of await builder.execute()) { - document.addData(sitePolygon.uuid, new SitePolygonDto(sitePolygon)); + document.addData( + sitePolygon.uuid, + new SitePolygonDto(sitePolygon, await this.sitePolygonService.convertIndicators(sitePolygon)) + ); } return document.serialize(); } diff --git a/apps/research-service/src/site-polygons/site-polygons.service.ts b/apps/research-service/src/site-polygons/site-polygons.service.ts index c52f55a..39f0933 100644 --- a/apps/research-service/src/site-polygons/site-polygons.service.ts +++ b/apps/research-service/src/site-polygons/site-polygons.service.ts @@ -1,9 +1,22 @@ -import { BadRequestException, Injectable } from "@nestjs/common"; +import { BadRequestException, Injectable, Type } from "@nestjs/common"; import { SitePolygon } from "@terramatch-microservices/database/entities"; import { Attributes, FindOptions, Op, WhereOptions } from "sequelize"; +import { IndicatorDto } from "./dto/site-polygon.dto"; +import { INDICATOR_DTOS } from "./dto/indicators.dto"; +import { ModelPropertiesAccessor } from "@nestjs/swagger/dist/services/model-properties-accessor"; +import { pick } from "lodash"; class SitePolygonQueryBuilder { - private findOptions: FindOptions> = {}; + private findOptions: FindOptions> = { + include: [ + "indicatorsFieldMonitoring", + "indicatorsHectares", + "indicatorsMsuCarbon", + "indicatorsTreeCount", + "indicatorsTreeCover", + "indicatorsTreeCoverLoss" + ] + }; constructor(pageSize: number) { this.findOptions.limit = pageSize; @@ -33,4 +46,16 @@ export class SitePolygonsService { if (pageAfter != null) await builder.pageAfter(pageAfter); return builder; } + + async convertIndicators(sitePolygon: SitePolygon): Promise { + const accessor = new ModelPropertiesAccessor(); + const indicators: IndicatorDto[] = []; + for (const indicator of await sitePolygon.getIndicators()) { + const DtoPrototype = INDICATOR_DTOS[indicator.indicatorSlug]; + const fields = accessor.getModelProperties(DtoPrototype as unknown as Type); + indicators.push(pick(indicator, fields) as typeof DtoPrototype); + } + + return indicators; + } } diff --git a/libs/database/src/lib/constants/index.ts b/libs/database/src/lib/constants/index.ts index a1212e1..3dd1c35 100644 --- a/libs/database/src/lib/constants/index.ts +++ b/libs/database/src/lib/constants/index.ts @@ -1 +1,2 @@ +export * from "./polygon-indicators"; export * from "./polygon-status"; diff --git a/libs/database/src/lib/constants/polygon-indicators.ts b/libs/database/src/lib/constants/polygon-indicators.ts new file mode 100644 index 0000000..2fb685a --- /dev/null +++ b/libs/database/src/lib/constants/polygon-indicators.ts @@ -0,0 +1,15 @@ +// Matches the indicators defined on https://gfw.atlassian.net/wiki/spaces/TerraMatch/pages/1469448210/Indicator+Data+Model +export const INDICATORS = { + 1: "treeCover", + 2: "treeCoverLoss", + 3: "treeCoverLossFires", + 4: "restorationByEcoRegion", + 5: "restorationByStrategy", + 6: "restorationByLandUse", + 7: "treeCount", + 8: "earlyTreeVerification", + 9: "fieldMonitoring", + 10: "msuCarbon" +} as const; +export const INDICATOR_SLUGS = Object.values(INDICATORS); +export type IndicatorSlug = (typeof INDICATOR_SLUGS)[number]; diff --git a/libs/database/src/lib/entities/index.ts b/libs/database/src/lib/entities/index.ts index 704f06b..c780e05 100644 --- a/libs/database/src/lib/entities/index.ts +++ b/libs/database/src/lib/entities/index.ts @@ -1,6 +1,12 @@ export * from "./delayed-job.entity"; export * from "./framework.entity"; export * from "./framework-user.entity"; +export * from "./indicator-output-field-monitoring.entity"; +export * from "./indicator-output-hectares.entity"; +export * from "./indicator-output-msu-carbon.entity"; +export * from "./indicator-output-tree-count.entity"; +export * from "./indicator-output-tree-cover.entity"; +export * from "./indicator-output-tree-cover-loss.entity"; export * from "./model-has-role.entity"; export * from "./organisation.entity"; export * from "./organisation-user.entity"; diff --git a/libs/database/src/lib/entities/indicator-output-field-monitoring.entity.ts b/libs/database/src/lib/entities/indicator-output-field-monitoring.entity.ts new file mode 100644 index 0000000..4f8f5ea --- /dev/null +++ b/libs/database/src/lib/entities/indicator-output-field-monitoring.entity.ts @@ -0,0 +1,41 @@ +import { AllowNull, AutoIncrement, Column, ForeignKey, Model, PrimaryKey, Table, Unique } from "sequelize-typescript"; +import { BIGINT, INTEGER, STRING } from "sequelize"; +import { SitePolygon } from "./site-polygon.entity"; +import { INDICATOR_SLUGS, IndicatorSlug } from "../constants"; + +@Table({ tableName: "indicator_output_field_monitoring", underscored: true, paranoid: true }) +export class IndicatorOutputFieldMonitoring extends Model { + @PrimaryKey + @AutoIncrement + @Column(BIGINT.UNSIGNED) + override id: number; + + @Unique("unique_polygon_indicator_year") + @ForeignKey(() => SitePolygon) + @Column({ type: BIGINT.UNSIGNED }) + sitePolygonId: number; + + @Unique("unique_polygon_indicator_year") + @Column({ type: STRING, values: INDICATOR_SLUGS }) + indicatorSlug: IndicatorSlug; + + @Unique("unique_polygon_indicator_year") + @Column(INTEGER) + yearOfAnalysis: number; + + @AllowNull + @Column(INTEGER) + treeCount: number | null; + + @AllowNull + @Column(STRING) + projectPhase: string | null; + + @AllowNull + @Column(STRING) + species: string | null; + + @AllowNull + @Column(INTEGER) + survivalRate: number | null; +} diff --git a/libs/database/src/lib/entities/indicator-output-hectares.entity.ts b/libs/database/src/lib/entities/indicator-output-hectares.entity.ts new file mode 100644 index 0000000..d1a8610 --- /dev/null +++ b/libs/database/src/lib/entities/indicator-output-hectares.entity.ts @@ -0,0 +1,28 @@ +import { AutoIncrement, Column, ForeignKey, Model, PrimaryKey, Table, Unique } from "sequelize-typescript"; +import { BIGINT, JSON, INTEGER, STRING } from "sequelize"; +import { SitePolygon } from "./site-polygon.entity"; +import { INDICATOR_SLUGS, IndicatorSlug } from "../constants"; + +@Table({ tableName: "indicator_output_hectares", underscored: true, paranoid: true }) +export class IndicatorOutputHectares extends Model { + @PrimaryKey + @AutoIncrement + @Column(BIGINT.UNSIGNED) + override id: number; + + @Unique("unique_polygon_indicator_year") + @ForeignKey(() => SitePolygon) + @Column({ type: BIGINT.UNSIGNED }) + sitePolygonId: number; + + @Unique("unique_polygon_indicator_year") + @Column({ type: STRING, values: INDICATOR_SLUGS }) + indicatorSlug: IndicatorSlug; + + @Unique("unique_polygon_indicator_year") + @Column(INTEGER) + yearOfAnalysis: number; + + @Column(JSON) + value: object; +} diff --git a/libs/database/src/lib/entities/indicator-output-msu-carbon.entity.ts b/libs/database/src/lib/entities/indicator-output-msu-carbon.entity.ts new file mode 100644 index 0000000..052bc0d --- /dev/null +++ b/libs/database/src/lib/entities/indicator-output-msu-carbon.entity.ts @@ -0,0 +1,37 @@ +import { AllowNull, AutoIncrement, Column, ForeignKey, Model, PrimaryKey, Table, Unique } from "sequelize-typescript"; +import { BIGINT, INTEGER, STRING } from "sequelize"; +import { SitePolygon } from "./site-polygon.entity"; +import { INDICATOR_SLUGS, IndicatorSlug } from "../constants"; + +@Table({ tableName: "indicator_output_msu_carbon", underscored: true, paranoid: true }) +export class IndicatorOutputMsuCarbon extends Model { + @PrimaryKey + @AutoIncrement + @Column(BIGINT.UNSIGNED) + override id: number; + + @Unique("unique_polygon_indicator_year") + @ForeignKey(() => SitePolygon) + @Column({ type: BIGINT.UNSIGNED }) + sitePolygonId: number; + + @Unique("unique_polygon_indicator_year") + @Column({ type: STRING, values: INDICATOR_SLUGS }) + indicatorSlug: IndicatorSlug; + + @Unique("unique_polygon_indicator_year") + @Column(INTEGER) + yearOfAnalysis: number; + + @AllowNull + @Column({ type: INTEGER, field: "carbon_ouput" }) + carbonOutput: number | null; + + @AllowNull + @Column(STRING) + projectPhase: string | null; + + @AllowNull + @Column(INTEGER) + confidence: number | null; +} diff --git a/libs/database/src/lib/entities/indicator-output-tree-count.entity.ts b/libs/database/src/lib/entities/indicator-output-tree-count.entity.ts new file mode 100644 index 0000000..a3e308e --- /dev/null +++ b/libs/database/src/lib/entities/indicator-output-tree-count.entity.ts @@ -0,0 +1,61 @@ +import { AllowNull, AutoIncrement, Column, ForeignKey, Model, PrimaryKey, Table, Unique } from "sequelize-typescript"; +import { BIGINT, DATE, INTEGER, STRING } from "sequelize"; +import { SitePolygon } from "./site-polygon.entity"; +import { INDICATOR_SLUGS, IndicatorSlug } from "../constants"; + +@Table({ tableName: "indicator_output_tree_count", underscored: true, paranoid: true }) +export class IndicatorOutputTreeCount extends Model { + @PrimaryKey + @AutoIncrement + @Column(BIGINT.UNSIGNED) + override id: number; + + @Unique("unique_polygon_indicator_year") + @ForeignKey(() => SitePolygon) + @Column({ type: BIGINT.UNSIGNED }) + sitePolygonId: number; + + @Unique("unique_polygon_indicator_year") + @Column({ type: STRING, values: INDICATOR_SLUGS }) + indicatorSlug: IndicatorSlug; + + @Unique("unique_polygon_indicator_year") + @Column(INTEGER) + yearOfAnalysis: number; + + @AllowNull + @Column(STRING) + surveyType: string | null; + + @AllowNull + @Column(INTEGER) + surveyId: number | null; + + @AllowNull + @Column(INTEGER) + treeCount: number | null; + + @AllowNull + @Column(STRING) + uncertaintyType: string | null; + + @AllowNull + @Column(STRING) + imagerySource: string | null; + + @AllowNull + @Column(DATE) + collectionDate: Date | null; + + @AllowNull + @Column(STRING) + imageryId: string | null; + + @AllowNull + @Column(STRING) + projectPhase: string | null; + + @AllowNull + @Column(INTEGER) + confidence: number | null; +} diff --git a/libs/database/src/lib/entities/indicator-output-tree-cover-loss.entity.ts b/libs/database/src/lib/entities/indicator-output-tree-cover-loss.entity.ts new file mode 100644 index 0000000..6c93af5 --- /dev/null +++ b/libs/database/src/lib/entities/indicator-output-tree-cover-loss.entity.ts @@ -0,0 +1,28 @@ +import { AutoIncrement, Column, ForeignKey, Model, PrimaryKey, Table, Unique } from "sequelize-typescript"; +import { BIGINT, INTEGER, JSON, STRING } from "sequelize"; +import { SitePolygon } from "./site-polygon.entity"; +import { INDICATOR_SLUGS, IndicatorSlug } from "../constants"; + +@Table({ tableName: "indicator_output_tree_cover_loss", underscored: true, paranoid: true }) +export class IndicatorOutputTreeCoverLoss extends Model { + @PrimaryKey + @AutoIncrement + @Column(BIGINT.UNSIGNED) + override id: number; + + @Unique("unique_polygon_indicator_year") + @ForeignKey(() => SitePolygon) + @Column({ type: BIGINT.UNSIGNED }) + sitePolygonId: number; + + @Unique("unique_polygon_indicator_year") + @Column({ type: STRING, values: INDICATOR_SLUGS }) + indicatorSlug: IndicatorSlug; + + @Unique("unique_polygon_indicator_year") + @Column(INTEGER) + yearOfAnalysis: number; + + @Column(JSON) + value: object; +} diff --git a/libs/database/src/lib/entities/indicator-output-tree-cover.entity.ts b/libs/database/src/lib/entities/indicator-output-tree-cover.entity.ts new file mode 100644 index 0000000..12c867f --- /dev/null +++ b/libs/database/src/lib/entities/indicator-output-tree-cover.entity.ts @@ -0,0 +1,37 @@ +import { AllowNull, AutoIncrement, Column, ForeignKey, Model, PrimaryKey, Table, Unique } from "sequelize-typescript"; +import { BIGINT, INTEGER, STRING } from "sequelize"; +import { SitePolygon } from "./site-polygon.entity"; +import { INDICATOR_SLUGS, IndicatorSlug } from "../constants"; + +@Table({ tableName: "indicator_output_tree_cover", underscored: true, paranoid: true }) +export class IndicatorOutputTreeCover extends Model { + @PrimaryKey + @AutoIncrement + @Column(BIGINT.UNSIGNED) + override id: number; + + @Unique("unique_polygon_indicator_year") + @ForeignKey(() => SitePolygon) + @Column({ type: BIGINT.UNSIGNED }) + sitePolygonId: number; + + @Unique("unique_polygon_indicator_year") + @Column({ type: STRING, values: INDICATOR_SLUGS }) + indicatorSlug: IndicatorSlug; + + @Unique("unique_polygon_indicator_year") + @Column(INTEGER) + yearOfAnalysis: number; + + @AllowNull + @Column(INTEGER) + percentCover: number | null; + + @AllowNull + @Column(STRING) + projectPhase: string | null; + + @AllowNull + @Column(INTEGER) + plusMinusPercent: number | null; +} diff --git a/libs/database/src/lib/entities/site-polygon.entity.ts b/libs/database/src/lib/entities/site-polygon.entity.ts index 9fb3d76..9469237 100644 --- a/libs/database/src/lib/entities/site-polygon.entity.ts +++ b/libs/database/src/lib/entities/site-polygon.entity.ts @@ -5,6 +5,7 @@ import { Column, Default, ForeignKey, + HasMany, Index, Model, PrimaryKey, @@ -15,7 +16,21 @@ import { Site } from "./site.entity"; import { PointGeometry } from "./point-geometry.entity"; import { PolygonGeometry } from "./polygon-geometry.entity"; import { User } from "./user.entity"; -import { POLYGON_STATUSES, PolygonStatus } from "../constants"; +import { INDICATOR_SLUGS, POLYGON_STATUSES, PolygonStatus } from "../constants"; +import { IndicatorOutputFieldMonitoring } from "./indicator-output-field-monitoring.entity"; +import { IndicatorOutputHectares } from "./indicator-output-hectares.entity"; +import { IndicatorOutputMsuCarbon } from "./indicator-output-msu-carbon.entity"; +import { IndicatorOutputTreeCount } from "./indicator-output-tree-count.entity"; +import { IndicatorOutputTreeCover } from "./indicator-output-tree-cover.entity"; +import { IndicatorOutputTreeCoverLoss } from "./indicator-output-tree-cover-loss.entity"; + +export type Indicator = + | IndicatorOutputTreeCoverLoss + | IndicatorOutputHectares + | IndicatorOutputTreeCount + | IndicatorOutputTreeCover + | IndicatorOutputFieldMonitoring + | IndicatorOutputMsuCarbon; @Table({ tableName: "site_polygon", underscored: true, paranoid: true }) export class SitePolygon extends Model { @@ -128,4 +143,62 @@ export class SitePolygon extends Model { @AllowNull @Column(STRING) versionName: string | null; + + @HasMany(() => IndicatorOutputFieldMonitoring) + indicatorsFieldMonitoring: IndicatorOutputFieldMonitoring[] | null; + + @HasMany(() => IndicatorOutputHectares) + indicatorsHectares: IndicatorOutputHectares[] | null; + + @HasMany(() => IndicatorOutputMsuCarbon) + indicatorsMsuCarbon: IndicatorOutputMsuCarbon[] | null; + + @HasMany(() => IndicatorOutputTreeCount) + indicatorsTreeCount: IndicatorOutputTreeCount[] | null; + + @HasMany(() => IndicatorOutputTreeCover) + indicatorsTreeCover: IndicatorOutputTreeCover[] | null; + + @HasMany(() => IndicatorOutputTreeCoverLoss) + indicatorsTreeCoverLoss: IndicatorOutputTreeCoverLoss[] | null; + + private _indicators: Indicator[] | null; + async getIndicators(refresh = false) { + if (!refresh && this._indicators != null) return this._indicators; + + if (refresh || this.indicatorsFieldMonitoring == null) { + this.indicatorsFieldMonitoring = await this.$get("indicatorsFieldMonitoring"); + } + if (refresh || this.indicatorsHectares == null) { + this.indicatorsHectares = await this.$get("indicatorsHectares"); + } + if (refresh || this.indicatorsMsuCarbon == null) { + this.indicatorsMsuCarbon = await this.$get("indicatorsMsuCarbon"); + } + if (refresh || this.indicatorsTreeCount == null) { + this.indicatorsTreeCount = await this.$get("indicatorsTreeCount"); + } + if (refresh || this.indicatorsTreeCover == null) { + this.indicatorsTreeCover = await this.$get("indicatorsTreeCover"); + } + if (refresh || this.indicatorsTreeCoverLoss == null) { + this.indicatorsTreeCoverLoss = await this.$get("indicatorsTreeCoverLoss"); + } + + this._indicators = [ + ...(this.indicatorsFieldMonitoring ?? []), + ...(this.indicatorsHectares ?? []), + ...(this.indicatorsMsuCarbon ?? []), + ...(this.indicatorsTreeCount ?? []), + ...(this.indicatorsTreeCover ?? []), + ...(this.indicatorsTreeCoverLoss ?? []) + ]; + this._indicators.sort((indicatorA, indicatorB) => { + const indexA = INDICATOR_SLUGS.indexOf(indicatorA.indicatorSlug); + const indexB = INDICATOR_SLUGS.indexOf(indicatorB.indicatorSlug); + return indexA < indexB ? -1 : indexB < indexA ? 1 : 0; + }); + + return this._indicators; + } } From 57396fea46233fccf2a84416a42309f23fbbc62f Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Thu, 7 Nov 2024 10:13:42 -0800 Subject: [PATCH 15/41] [TM-1451] Work around a sequelize bug. --- .../lib/entities/indicator-output-hectares.entity.ts | 12 ++++++++++-- .../indicator-output-tree-cover-loss.entity.ts | 12 ++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/libs/database/src/lib/entities/indicator-output-hectares.entity.ts b/libs/database/src/lib/entities/indicator-output-hectares.entity.ts index d1a8610..076c2b0 100644 --- a/libs/database/src/lib/entities/indicator-output-hectares.entity.ts +++ b/libs/database/src/lib/entities/indicator-output-hectares.entity.ts @@ -1,5 +1,5 @@ import { AutoIncrement, Column, ForeignKey, Model, PrimaryKey, Table, Unique } from "sequelize-typescript"; -import { BIGINT, JSON, INTEGER, STRING } from "sequelize"; +import { BIGINT, INTEGER, JSON as JSON_TYPE, STRING } from "sequelize"; import { SitePolygon } from "./site-polygon.entity"; import { INDICATOR_SLUGS, IndicatorSlug } from "../constants"; @@ -23,6 +23,14 @@ export class IndicatorOutputHectares extends Model { @Column(INTEGER) yearOfAnalysis: number; - @Column(JSON) + @Column({ + type: JSON_TYPE, + // Sequelize has a bug where when the data for this model is fetched as part of an include on + // findAll, the JSON value isn't getting deserialized. + get(this: IndicatorOutputHectares): object { + const value = this.getDataValue("value"); + return typeof value === "string" ? JSON.parse(value) : value; + } + }) value: object; } diff --git a/libs/database/src/lib/entities/indicator-output-tree-cover-loss.entity.ts b/libs/database/src/lib/entities/indicator-output-tree-cover-loss.entity.ts index 6c93af5..123abec 100644 --- a/libs/database/src/lib/entities/indicator-output-tree-cover-loss.entity.ts +++ b/libs/database/src/lib/entities/indicator-output-tree-cover-loss.entity.ts @@ -1,5 +1,5 @@ import { AutoIncrement, Column, ForeignKey, Model, PrimaryKey, Table, Unique } from "sequelize-typescript"; -import { BIGINT, INTEGER, JSON, STRING } from "sequelize"; +import { BIGINT, INTEGER, JSON as JSON_TYPE, STRING } from "sequelize"; import { SitePolygon } from "./site-polygon.entity"; import { INDICATOR_SLUGS, IndicatorSlug } from "../constants"; @@ -23,6 +23,14 @@ export class IndicatorOutputTreeCoverLoss extends Model { @Column(INTEGER) yearOfAnalysis: number; - @Column(JSON) + @Column({ + type: JSON_TYPE, + // Sequelize has a bug where when the data for this model is fetched as part of an include on + // findAll, the JSON value isn't getting deserialized. + get(this: IndicatorOutputTreeCoverLoss): object { + const value = this.getDataValue("value"); + return typeof value === "string" ? JSON.parse(value) : value; + } + }) value: object; } From 8778baa31e65d6b91807fbe7920c9cc65dd59e80 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Thu, 7 Nov 2024 10:23:06 -0800 Subject: [PATCH 16/41] [TM-1451] Make the JsonApiAttributes constructor more useful. --- .../src/jobs/dto/delayed-job.dto.ts | 6 --- .../src/site-polygons/dto/site-polygon.dto.ts | 9 +---- .../common/src/lib/dto/json-api-attributes.ts | 9 ++++- libs/common/src/lib/dto/organisation.dto.ts | 18 +++------ libs/common/src/lib/dto/user.dto.ts | 37 ++++++++----------- 5 files changed, 28 insertions(+), 51 deletions(-) diff --git a/apps/job-service/src/jobs/dto/delayed-job.dto.ts b/apps/job-service/src/jobs/dto/delayed-job.dto.ts index 91b7ba6..8fbab5d 100644 --- a/apps/job-service/src/jobs/dto/delayed-job.dto.ts +++ b/apps/job-service/src/jobs/dto/delayed-job.dto.ts @@ -1,18 +1,12 @@ import { JsonApiAttributes } from "@terramatch-microservices/common/dto/json-api-attributes"; import { JsonApiDto } from "@terramatch-microservices/common/decorators"; import { ApiProperty } from "@nestjs/swagger"; -import { DelayedJob } from "@terramatch-microservices/database/entities"; const STATUSES = ["pending", "failed", "succeeded"]; type Status = (typeof STATUSES)[number]; @JsonApiDto({ type: "delayedJobs" }) export class DelayedJobDto extends JsonApiAttributes { - constructor(job: DelayedJob) { - const { status, statusCode, payload } = job; - super({ status, statusCode, payload }); - } - @ApiProperty({ description: "The current status of the job. If the status is not pending, the payload and statusCode will be provided.", diff --git a/apps/research-service/src/site-polygons/dto/site-polygon.dto.ts b/apps/research-service/src/site-polygons/dto/site-polygon.dto.ts index 8e3c553..825fe93 100644 --- a/apps/research-service/src/site-polygons/dto/site-polygon.dto.ts +++ b/apps/research-service/src/site-polygons/dto/site-polygon.dto.ts @@ -47,16 +47,9 @@ class ReportingPeriod { export class SitePolygonDto extends JsonApiAttributes { constructor(sitePolygon: SitePolygon, indicators: IndicatorDto[]) { super({ + ...sitePolygon, name: sitePolygon.polyName, - status: sitePolygon.status, siteId: sitePolygon.siteUuid, - plantStart: sitePolygon.plantStart, - plantEnd: sitePolygon.plantEnd, - practice: sitePolygon.practice, - targetSys: sitePolygon.targetSys, - distr: sitePolygon.distr, - numTrees: sitePolygon.numTrees, - calcArea: sitePolygon.calcArea, indicators, establishmentTreeSpecies: [], reportingPeriods: [] diff --git a/libs/common/src/lib/dto/json-api-attributes.ts b/libs/common/src/lib/dto/json-api-attributes.ts index 94ea2db..66a02ed 100644 --- a/libs/common/src/lib/dto/json-api-attributes.ts +++ b/libs/common/src/lib/dto/json-api-attributes.ts @@ -1,3 +1,6 @@ +import { ModelPropertiesAccessor } from "@nestjs/swagger/dist/services/model-properties-accessor"; +import { pick } from "lodash"; + /** * A simple class to make it easy to create a typed attributes DTO with new() * @@ -5,7 +8,9 @@ * See auth.controller.ts login for a simple example. */ export class JsonApiAttributes { - constructor(props: Omit) { - Object.assign(this, props); + constructor(source: Omit) { + // This assigns only the attributes from source that are defined as ApiProperty in this DTO. + const accessor = new ModelPropertiesAccessor(); + Object.assign(this, pick(source, accessor.getModelProperties(this.constructor.prototype))); } } diff --git a/libs/common/src/lib/dto/organisation.dto.ts b/libs/common/src/lib/dto/organisation.dto.ts index 609858e..273a3b6 100644 --- a/libs/common/src/lib/dto/organisation.dto.ts +++ b/libs/common/src/lib/dto/organisation.dto.ts @@ -1,20 +1,12 @@ -import { JsonApiDto } from '../decorators'; -import { JsonApiAttributes } from './json-api-attributes'; -import { ApiProperty } from '@nestjs/swagger'; -import { Organisation } from '@terramatch-microservices/database/entities'; +import { JsonApiDto } from "../decorators"; +import { JsonApiAttributes } from "./json-api-attributes"; +import { ApiProperty } from "@nestjs/swagger"; -const STATUSES = ['draft', 'pending', 'approved', 'rejected']; +const STATUSES = ["draft", "pending", "approved", "rejected"]; type Status = (typeof STATUSES)[number]; -@JsonApiDto({ type: 'organisations' }) +@JsonApiDto({ type: "organisations" }) export class OrganisationDto extends JsonApiAttributes { - constructor(org: Organisation) { - super({ - status: org.status as Status, - name: org.name - }); - } - @ApiProperty({ enum: STATUSES }) status: Status; diff --git a/libs/common/src/lib/dto/user.dto.ts b/libs/common/src/lib/dto/user.dto.ts index bea8c69..e6872ab 100644 --- a/libs/common/src/lib/dto/user.dto.ts +++ b/libs/common/src/lib/dto/user.dto.ts @@ -1,33 +1,26 @@ -import { ApiProperty } from '@nestjs/swagger'; -import { JsonApiDto } from '../decorators'; -import { JsonApiAttributes } from './json-api-attributes'; -import { Framework, User } from '@terramatch-microservices/database/entities'; +import { ApiProperty } from "@nestjs/swagger"; +import { JsonApiDto } from "../decorators"; +import { JsonApiAttributes } from "./json-api-attributes"; +import { Framework, User } from "@terramatch-microservices/database/entities"; -class UserFramework { - @ApiProperty({ example: 'TerraFund Landscapes' }) +class UserFramework { + @ApiProperty({ example: "TerraFund Landscapes" }) name: string; - @ApiProperty({ example: 'terrafund-landscapes' }) + @ApiProperty({ example: "terrafund-landscapes" }) slug: string; } -@JsonApiDto({ type: 'users' }) +@JsonApiDto({ type: "users" }) export class UserDto extends JsonApiAttributes { constructor(user: User, frameworks: Framework[]) { super({ - uuid: user.uuid ?? '', - firstName: user.firstName, - lastName: user.lastName, - fullName: - user.firstName == null || user.lastName == null - ? null - : `${user.firstName} ${user.lastName}`, + ...user, + uuid: user.uuid ?? "", + fullName: user.firstName == null || user.lastName == null ? null : `${user.firstName} ${user.lastName}`, primaryRole: user.primaryRole, - emailAddress: user.emailAddress, - emailAddressVerifiedAt: user.emailAddressVerifiedAt, - locale: user.locale, frameworks: frameworks.map(({ name, slug }) => ({ name, slug })) - }) + }); } @ApiProperty() @@ -39,13 +32,13 @@ export class UserDto extends JsonApiAttributes { @ApiProperty({ nullable: true }) lastName: string | null; - @ApiProperty({ nullable: true, description: 'Currently just calculated by appending lastName to firstName.' }) + @ApiProperty({ nullable: true, description: "Currently just calculated by appending lastName to firstName." }) fullName: string | null; @ApiProperty() primaryRole: string; - @ApiProperty({ example: 'person@foocorp.net' }) + @ApiProperty({ example: "person@foocorp.net" }) emailAddress: string; @ApiProperty({ nullable: true }) @@ -55,5 +48,5 @@ export class UserDto extends JsonApiAttributes { locale: string | null; @ApiProperty({ type: () => UserFramework, isArray: true }) - frameworks: UserFramework[] + frameworks: UserFramework[]; } From 9ccfba4f537588959dbc8d451d16a0f1777325c6 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Thu, 7 Nov 2024 10:25:03 -0800 Subject: [PATCH 17/41] [TM-1451] Add a note about the new package scripts for service subsets. --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 7e18aaa..cae5048 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,8 @@ Repository for the Microservices API backend of the TerraMatch service * The default maximum number of services it can run in parallel is 3. To run all of the services at once, use something like `nx run-many --parallel=100 -t serve`, or you can cherry-pick which services you want to run instead with `nx run-many -t serve --projects user-service jobs-service`. + * Some useful targets have been added to the root `package.json` for service sets. For instance, to run just the services needed + by the TM React front end, use `npm run fe-services`, or to run all use `npm run all`. * In `.env` in your `wri-terramatch-website` repository, set your BE connection URL correctly by noting the config in `.env.local.sample` for local development. * The `NEXT_PUBLIC_API_BASE_URL` still points at the PHP BE directly From ee305a0a7470fe03f3ad23fd5cfc52490045e361 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Thu, 7 Nov 2024 10:44:15 -0800 Subject: [PATCH 18/41] [TM-1451] Fix JsonApiAttributes improvement. --- .../src/site-polygons/dto/site-polygon.dto.ts | 3 +- .../common/src/lib/dto/json-api-attributes.ts | 5 +- libs/common/src/lib/dto/user.dto.ts | 5 +- libs/database/src/lib/entities/user.entity.ts | 121 ++++++++---------- 4 files changed, 58 insertions(+), 76 deletions(-) diff --git a/apps/research-service/src/site-polygons/dto/site-polygon.dto.ts b/apps/research-service/src/site-polygons/dto/site-polygon.dto.ts index 825fe93..a6c33df 100644 --- a/apps/research-service/src/site-polygons/dto/site-polygon.dto.ts +++ b/apps/research-service/src/site-polygons/dto/site-polygon.dto.ts @@ -46,8 +46,7 @@ class ReportingPeriod { @JsonApiDto({ type: "sitePolygons" }) export class SitePolygonDto extends JsonApiAttributes { constructor(sitePolygon: SitePolygon, indicators: IndicatorDto[]) { - super({ - ...sitePolygon, + super(sitePolygon, { name: sitePolygon.polyName, siteId: sitePolygon.siteUuid, indicators, diff --git a/libs/common/src/lib/dto/json-api-attributes.ts b/libs/common/src/lib/dto/json-api-attributes.ts index 66a02ed..9cf1199 100644 --- a/libs/common/src/lib/dto/json-api-attributes.ts +++ b/libs/common/src/lib/dto/json-api-attributes.ts @@ -8,9 +8,12 @@ import { pick } from "lodash"; * See auth.controller.ts login for a simple example. */ export class JsonApiAttributes { - constructor(source: Omit) { + constructor(source: Partial>, overrides?: Partial>) { // This assigns only the attributes from source that are defined as ApiProperty in this DTO. const accessor = new ModelPropertiesAccessor(); Object.assign(this, pick(source, accessor.getModelProperties(this.constructor.prototype))); + if (overrides != null) { + Object.assign(this, pick(overrides, accessor.getModelProperties(this.constructor.prototype))); + } } } diff --git a/libs/common/src/lib/dto/user.dto.ts b/libs/common/src/lib/dto/user.dto.ts index e6872ab..8a1ddb3 100644 --- a/libs/common/src/lib/dto/user.dto.ts +++ b/libs/common/src/lib/dto/user.dto.ts @@ -14,11 +14,8 @@ class UserFramework { @JsonApiDto({ type: "users" }) export class UserDto extends JsonApiAttributes { constructor(user: User, frameworks: Framework[]) { - super({ - ...user, + super(user as Omit, { uuid: user.uuid ?? "", - fullName: user.firstName == null || user.lastName == null ? null : `${user.firstName} ${user.lastName}`, - primaryRole: user.primaryRole, frameworks: frameworks.map(({ name, slug }) => ({ name, slug })) }); } diff --git a/libs/database/src/lib/entities/user.entity.ts b/libs/database/src/lib/entities/user.entity.ts index d795151..ccc4858 100644 --- a/libs/database/src/lib/entities/user.entity.ts +++ b/libs/database/src/lib/entities/user.entity.ts @@ -1,4 +1,4 @@ -import { uniq } from 'lodash'; +import { uniq } from "lodash"; import { AllowNull, AutoIncrement, @@ -11,20 +11,20 @@ import { Model, PrimaryKey, Table, - Unique, -} from 'sequelize-typescript'; -import { BIGINT, BOOLEAN, col, DATE, fn, Op, STRING, UUID } from 'sequelize'; -import { Role } from './role.entity'; -import { ModelHasRole } from './model-has-role.entity'; -import { Permission } from './permission.entity'; -import { Framework } from './framework.entity'; -import { Project } from './project.entity'; -import { ProjectUser } from './project-user.entity'; -import { Organisation } from './organisation.entity'; -import { OrganisationUser } from './organisation-user.entity'; -import { FrameworkUser } from './framework-user.entity'; - -@Table({ tableName: 'users', underscored: true, paranoid: true }) + Unique +} from "sequelize-typescript"; +import { BIGINT, BOOLEAN, col, DATE, fn, Op, STRING, UUID } from "sequelize"; +import { Role } from "./role.entity"; +import { ModelHasRole } from "./model-has-role.entity"; +import { Permission } from "./permission.entity"; +import { Framework } from "./framework.entity"; +import { Project } from "./project.entity"; +import { ProjectUser } from "./project-user.entity"; +import { Organisation } from "./organisation.entity"; +import { OrganisationUser } from "./organisation-user.entity"; +import { FrameworkUser } from "./framework-user.entity"; + +@Table({ tableName: "users", underscored: true, paranoid: true }) export class User extends Model { @PrimaryKey @AutoIncrement @@ -128,19 +128,19 @@ export class User extends Model { locale: string; @BelongsToMany(() => Role, { - foreignKey: 'modelId', + foreignKey: "modelId", through: { model: () => ModelHasRole, unique: false, scope: { - modelType: 'App\\Models\\V2\\User', - }, - }, + modelType: "App\\Models\\V2\\User" + } + } }) roles: Role[]; async loadRoles() { - if (this.roles == null) this.roles = await (this as User).$get('roles'); + if (this.roles == null) this.roles = await (this as User).$get("roles"); return this.roles; } @@ -152,12 +152,16 @@ export class User extends Model { return this.roles?.[0]?.name; } + get fullName() { + return this.firstName == null || this.lastName == null ? null : `${this.firstName} ${this.lastName}`; + } + @BelongsToMany(() => Project, () => ProjectUser) projects: Project[]; async loadProjects() { if (this.projects == null) { - this.projects = await (this as User).$get('projects'); + this.projects = await (this as User).$get("projects"); } return this.projects; } @@ -167,7 +171,7 @@ export class User extends Model { async loadOrganisation() { if (this.organisation == null && this.organisationId != null) { - this.organisation = await (this as User).$get('organisation'); + this.organisation = await (this as User).$get("organisation"); } return this.organisation; } @@ -177,7 +181,7 @@ export class User extends Model { async loadOrganisations() { if (this.organisations == null) { - this.organisations = await (this as User).$get('organisations'); + this.organisations = await (this as User).$get("organisations"); } return this.organisations; } @@ -185,18 +189,14 @@ export class User extends Model { @BelongsToMany(() => Organisation, { through: { model: () => OrganisationUser, - scope: { status: 'approved' }, - }, + scope: { status: "approved" } + } }) - organisationsConfirmed: Array< - Organisation & { OrganisationUser: OrganisationUser } - >; + organisationsConfirmed: Array; async loadOrganisationsConfirmed() { if (this.organisationsConfirmed == null) { - this.organisationsConfirmed = await (this as User).$get( - 'organisationsConfirmed' - ); + this.organisationsConfirmed = await (this as User).$get("organisationsConfirmed"); } return this.organisationsConfirmed; } @@ -204,50 +204,38 @@ export class User extends Model { @BelongsToMany(() => Organisation, { through: { model: () => OrganisationUser, - scope: { status: 'requested' }, - }, + scope: { status: "requested" } + } }) - organisationsRequested: Array< - Organisation & { OrganisationUser: OrganisationUser } - >; + organisationsRequested: Array; async loadOrganisationsRequested() { if (this.organisationsRequested == null) { - this.organisationsRequested = await (this as User).$get( - 'organisationsRequested' - ); + this.organisationsRequested = await (this as User).$get("organisationsRequested"); } return this.organisationsRequested; } - private _primaryOrganisation: - | (Organisation & { OrganisationUser?: OrganisationUser }) - | false; - async primaryOrganisation(): Promise< - (Organisation & { OrganisationUser?: OrganisationUser }) | null - > { + private _primaryOrganisation: (Organisation & { OrganisationUser?: OrganisationUser }) | false; + async primaryOrganisation(): Promise<(Organisation & { OrganisationUser?: OrganisationUser }) | null> { if (this._primaryOrganisation == null) { await this.loadOrganisation(); if (this.organisation != null) { const userOrg = ( - await (this as User).$get('organisations', { + await (this as User).$get("organisations", { limit: 1, - where: { id: this.organisation.id }, + where: { id: this.organisation.id } }) )[0]; return (this._primaryOrganisation = userOrg ?? this.organisation); } - const confirmed = ( - await (this as User).$get('organisationsConfirmed', { limit: 1 }) - )[0]; + const confirmed = (await (this as User).$get("organisationsConfirmed", { limit: 1 }))[0]; if (confirmed != null) { return (this._primaryOrganisation = confirmed); } - const requested = ( - await (this as User).$get('organisationsRequested', { limit: 1 }) - )[0]; + const requested = (await (this as User).$get("organisationsRequested", { limit: 1 }))[0]; if (requested != null) { return (this._primaryOrganisation = requested); } @@ -255,9 +243,7 @@ export class User extends Model { this._primaryOrganisation = false; } - return this._primaryOrganisation === false - ? null - : this._primaryOrganisation; + return this._primaryOrganisation === false ? null : this._primaryOrganisation; } @BelongsToMany(() => Framework, () => FrameworkUser) @@ -265,7 +251,7 @@ export class User extends Model { async loadFrameworks() { if (this.frameworks == null) { - this.frameworks = await (this as User).$get('frameworks'); + this.frameworks = await (this as User).$get("frameworks"); } return this.frameworks; } @@ -274,8 +260,7 @@ export class User extends Model { async myFrameworks(): Promise { if (this._myFrameworks == null) { await this.loadRoles(); - const isAdmin = - this.roles.find(({ name }) => name.startsWith('admin-')) != null; + const isAdmin = this.roles.find(({ name }) => name.startsWith("admin-")) != null; await this.loadFrameworks(); @@ -283,25 +268,23 @@ export class User extends Model { if (isAdmin) { // Admins have access to all frameworks their permissions say they do const permissions = await Permission.getUserPermissionNames(this.id); - const prefix = 'framework-'; + const prefix = "framework-"; frameworkSlugs = [ ...frameworkSlugs, ...permissions - .filter((permission) => permission.startsWith(prefix)) - .map((permission) => permission.substring(prefix.length)), + .filter(permission => permission.startsWith(prefix)) + .map(permission => permission.substring(prefix.length)) ]; } else { // Other users have access to the frameworks embodied by their set of projects frameworkSlugs = [ ...frameworkSlugs, ...( - await (this as User).$get('projects', { - attributes: [ - [fn('DISTINCT', col('Project.framework_key')), 'frameworkKey'], - ], - raw: true, + await (this as User).$get("projects", { + attributes: [[fn("DISTINCT", col("Project.framework_key")), "frameworkKey"]], + raw: true }) - ).map(({ frameworkKey }) => frameworkKey), + ).map(({ frameworkKey }) => frameworkKey) ]; } @@ -309,7 +292,7 @@ export class User extends Model { frameworkSlugs = uniq(frameworkSlugs); return (this._myFrameworks = await Framework.findAll({ - where: { slug: { [Op.in]: frameworkSlugs } }, + where: { slug: { [Op.in]: frameworkSlugs } } })); } From 7ae8b0eae6444e82184faf9949a51a09427eef42 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Thu, 7 Nov 2024 12:50:41 -0800 Subject: [PATCH 19/41] [TM-1451] Tighten up the JsonApiAttributes improvement. --- .../src/site-polygons/dto/site-polygon.dto.ts | 5 +- .../common/src/lib/dto/json-api-attributes.ts | 66 +++++++++++++++++-- libs/common/src/lib/dto/user.dto.ts | 5 +- 3 files changed, 65 insertions(+), 11 deletions(-) diff --git a/apps/research-service/src/site-polygons/dto/site-polygon.dto.ts b/apps/research-service/src/site-polygons/dto/site-polygon.dto.ts index a6c33df..e67dedd 100644 --- a/apps/research-service/src/site-polygons/dto/site-polygon.dto.ts +++ b/apps/research-service/src/site-polygons/dto/site-polygon.dto.ts @@ -1,4 +1,4 @@ -import { JsonApiAttributes } from "@terramatch-microservices/common/dto/json-api-attributes"; +import { JsonApiAttributes, pickApiProperties } from "@terramatch-microservices/common/dto/json-api-attributes"; import { JsonApiDto } from "@terramatch-microservices/common/decorators"; import { ApiProperty } from "@nestjs/swagger"; import { @@ -46,7 +46,8 @@ class ReportingPeriod { @JsonApiDto({ type: "sitePolygons" }) export class SitePolygonDto extends JsonApiAttributes { constructor(sitePolygon: SitePolygon, indicators: IndicatorDto[]) { - super(sitePolygon, { + super({ + ...pickApiProperties(sitePolygon, SitePolygonDto), name: sitePolygon.polyName, siteId: sitePolygon.siteUuid, indicators, diff --git a/libs/common/src/lib/dto/json-api-attributes.ts b/libs/common/src/lib/dto/json-api-attributes.ts index 9cf1199..440c284 100644 --- a/libs/common/src/lib/dto/json-api-attributes.ts +++ b/libs/common/src/lib/dto/json-api-attributes.ts @@ -1,5 +1,62 @@ import { ModelPropertiesAccessor } from "@nestjs/swagger/dist/services/model-properties-accessor"; import { pick } from "lodash"; +import { Type } from "@nestjs/common"; + +// Some type shenanigans to represent a type that _only_ includes properties defined in the +// included union type. This implementation was difficult to track down and get working. Found +// explanation here: https://nabeelvalley.co.za/blog/2022/08-07/common-object-type/ +type CommonKeys = R extends T ? keyof T & CommonKeys> : keyof T; +type Common = Pick>; + +/** + * Returns an object with only the properties from source that are marked with @ApiProperty in the DTO. + * + * The return object will include all properties that exist on the source object and are defined + * in the DTO. However, the return type will only indicate that the properties that are common between + * the types passed in are present. This is useful to make sure that all properties that are expected + * are included in a given API response. + * + * This utility will also pull values from getters on objects as well as defined properties. + * + * Example from user.dto.ts: + * constructor(user: User, frameworks: Framework[]) { + * super({ + * ...pickApiProperties(user as Omit, UserDto), + * uuid: user.uuid ?? "", + * frameworks: frameworks.map(({ name, slug }) => ({ name, slug })) + * }); + * } + * + * In the example above, the type passed to pickApiProperties removes "uuid" and "frameworks" from + * the source type passed in, requiring that they be implemented in the full object that gets passed + * to super. + * + * Example from site-polygon.dto.ts: + * constructor(sitePolygon: SitePolygon, indicators: IndicatorDto[]) { + * super({ + * ...pickApiProperties(sitePolygon, SitePolygonDto), + * name: sitePolygon.polyName, + * siteId: sitePolygon.siteUuid, + * indicators, + * establishmentTreeSpecies: [], + * reportingPeriods: [] + * }); + * } + * + * In this example, the additional properties added are ones that exist on the DTO definition but + * not on the SitePolygon entity class. Since the super() call requires all the properties that + * are defined in the DTO, this structure will fail to compile if any of the additional props are + * missing. + * + * Note that if ...sitePolygon were used instead of ...pickApiProperties(sitePolygon, SitePolygonDto), + * we would fail to include properties that are accessed via getters (which turns out to include all + * data values on Sequelize objects), and would include anything extra is defined on sitePolygon. + */ +export function pickApiProperties(source: Source, dtoClass: Type) { + const accessor = new ModelPropertiesAccessor(); + const fields = accessor.getModelProperties(dtoClass.prototype); + return pick(source, fields) as Common; +} /** * A simple class to make it easy to create a typed attributes DTO with new() @@ -8,12 +65,7 @@ import { pick } from "lodash"; * See auth.controller.ts login for a simple example. */ export class JsonApiAttributes { - constructor(source: Partial>, overrides?: Partial>) { - // This assigns only the attributes from source that are defined as ApiProperty in this DTO. - const accessor = new ModelPropertiesAccessor(); - Object.assign(this, pick(source, accessor.getModelProperties(this.constructor.prototype))); - if (overrides != null) { - Object.assign(this, pick(overrides, accessor.getModelProperties(this.constructor.prototype))); - } + constructor(source: Omit) { + Object.assign(this, pickApiProperties(source, this.constructor as Type)); } } diff --git a/libs/common/src/lib/dto/user.dto.ts b/libs/common/src/lib/dto/user.dto.ts index 8a1ddb3..4bfcc98 100644 --- a/libs/common/src/lib/dto/user.dto.ts +++ b/libs/common/src/lib/dto/user.dto.ts @@ -1,6 +1,6 @@ import { ApiProperty } from "@nestjs/swagger"; import { JsonApiDto } from "../decorators"; -import { JsonApiAttributes } from "./json-api-attributes"; +import { JsonApiAttributes, pickApiProperties } from "./json-api-attributes"; import { Framework, User } from "@terramatch-microservices/database/entities"; class UserFramework { @@ -14,7 +14,8 @@ class UserFramework { @JsonApiDto({ type: "users" }) export class UserDto extends JsonApiAttributes { constructor(user: User, frameworks: Framework[]) { - super(user as Omit, { + super({ + ...pickApiProperties(user as Omit, UserDto), uuid: user.uuid ?? "", frameworks: frameworks.map(({ name, slug }) => ({ name, slug })) }); From 9377bd2b7fd7ab10a5cf0b02502b113b4603e856 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Thu, 7 Nov 2024 14:02:32 -0800 Subject: [PATCH 20/41] [TM-1451] Implement site establishment tree species. --- .../src/site-polygons/dto/site-polygon.dto.ts | 32 +++++------ .../site-polygons/site-polygons.controller.ts | 7 +-- .../site-polygons/site-polygons.service.ts | 14 ++++- libs/database/src/lib/entities/index.ts | 1 + libs/database/src/lib/entities/site.entity.ts | 18 +++++- .../src/lib/entities/tree-species.entity.ts | 55 +++++++++++++++++++ 6 files changed, 102 insertions(+), 25 deletions(-) create mode 100644 libs/database/src/lib/entities/tree-species.entity.ts diff --git a/apps/research-service/src/site-polygons/dto/site-polygon.dto.ts b/apps/research-service/src/site-polygons/dto/site-polygon.dto.ts index e67dedd..dd15c7e 100644 --- a/apps/research-service/src/site-polygons/dto/site-polygon.dto.ts +++ b/apps/research-service/src/site-polygons/dto/site-polygon.dto.ts @@ -20,12 +20,12 @@ export type IndicatorDto = | IndicatorFieldMonitoringDto | IndicatorMsuCarbonDto; -class TreeSpecies { +export class TreeSpeciesDto { @ApiProperty({ example: "Acacia binervia" }) name: string; - @ApiProperty({ example: 15000 }) - amount: number; + @ApiProperty({ example: 15000, nullable: true }) + amount: number | null; } class ReportingPeriod { @@ -36,22 +36,22 @@ class ReportingPeriod { submittedAt: Date; @ApiProperty({ - type: () => TreeSpecies, + type: () => TreeSpeciesDto, isArray: true, description: "The tree species reported as planted during this reporting period" }) - treeSpecies: TreeSpecies[]; + treeSpecies: TreeSpeciesDto[]; } @JsonApiDto({ type: "sitePolygons" }) export class SitePolygonDto extends JsonApiAttributes { - constructor(sitePolygon: SitePolygon, indicators: IndicatorDto[]) { + constructor(sitePolygon: SitePolygon, indicators: IndicatorDto[], establishmentTreeSpecies: TreeSpeciesDto[]) { super({ ...pickApiProperties(sitePolygon, SitePolygonDto), name: sitePolygon.polyName, siteId: sitePolygon.siteUuid, indicators, - establishmentTreeSpecies: [], + establishmentTreeSpecies, reportingPeriods: [] }); } @@ -65,25 +65,25 @@ export class SitePolygonDto extends JsonApiAttributes { @ApiProperty() siteId: string; - @ApiProperty() + @ApiProperty({ nullable: true }) plantStart: Date | null; - @ApiProperty() + @ApiProperty({ nullable: true }) plantEnd: Date | null; - @ApiProperty() + @ApiProperty({ nullable: true }) practice: string | null; - @ApiProperty() + @ApiProperty({ nullable: true }) targetSys: string | null; - @ApiProperty() + @ApiProperty({ nullable: true }) distr: string | null; - @ApiProperty() + @ApiProperty({ nullable: true }) numTrees: number | null; - @ApiProperty() + @ApiProperty({ nullable: true }) calcArea: number | null; @ApiProperty({ @@ -103,11 +103,11 @@ export class SitePolygonDto extends JsonApiAttributes { indicators: IndicatorDto[]; @ApiProperty({ - type: () => TreeSpecies, + type: () => TreeSpeciesDto, isArray: true, description: "The tree species associated with the establishment of the site that this polygon relates to." }) - establishmentTreeSpecies: TreeSpecies[]; + establishmentTreeSpecies: TreeSpeciesDto[]; @ApiProperty({ type: () => ReportingPeriod, diff --git a/apps/research-service/src/site-polygons/site-polygons.controller.ts b/apps/research-service/src/site-polygons/site-polygons.controller.ts index dd3df8c..dd10ebe 100644 --- a/apps/research-service/src/site-polygons/site-polygons.controller.ts +++ b/apps/research-service/src/site-polygons/site-polygons.controller.ts @@ -54,10 +54,9 @@ export class SitePolygonsController { const document = buildJsonApi(); for (const sitePolygon of await builder.execute()) { - document.addData( - sitePolygon.uuid, - new SitePolygonDto(sitePolygon, await this.sitePolygonService.convertIndicators(sitePolygon)) - ); + const indicators = await this.sitePolygonService.convertIndicators(sitePolygon); + const establishmentTreeSpecies = await this.sitePolygonService.getEstablishmentTreeSpecies(sitePolygon); + document.addData(sitePolygon.uuid, new SitePolygonDto(sitePolygon, indicators, establishmentTreeSpecies)); } return document.serialize(); } diff --git a/apps/research-service/src/site-polygons/site-polygons.service.ts b/apps/research-service/src/site-polygons/site-polygons.service.ts index 39f0933..6a68bcb 100644 --- a/apps/research-service/src/site-polygons/site-polygons.service.ts +++ b/apps/research-service/src/site-polygons/site-polygons.service.ts @@ -1,7 +1,7 @@ import { BadRequestException, Injectable, Type } from "@nestjs/common"; -import { SitePolygon } from "@terramatch-microservices/database/entities"; +import { Site, SitePolygon } from "@terramatch-microservices/database/entities"; import { Attributes, FindOptions, Op, WhereOptions } from "sequelize"; -import { IndicatorDto } from "./dto/site-polygon.dto"; +import { IndicatorDto, TreeSpeciesDto } from "./dto/site-polygon.dto"; import { INDICATOR_DTOS } from "./dto/indicators.dto"; import { ModelPropertiesAccessor } from "@nestjs/swagger/dist/services/model-properties-accessor"; import { pick } from "lodash"; @@ -14,7 +14,8 @@ class SitePolygonQueryBuilder { "indicatorsMsuCarbon", "indicatorsTreeCount", "indicatorsTreeCover", - "indicatorsTreeCoverLoss" + "indicatorsTreeCoverLoss", + { model: Site, include: ["treeSpecies"] } ] }; @@ -58,4 +59,11 @@ export class SitePolygonsService { return indicators; } + + async getEstablishmentTreeSpecies(sitePolygon: SitePolygon): Promise { + // These associations are expected to be eager loaded, so this should not result in new SQL + // queries. + const site = await sitePolygon.loadSite(); + return (await site.loadTreeSpecies()).map(({ name, amount }) => ({ name, amount })); + } } diff --git a/libs/database/src/lib/entities/index.ts b/libs/database/src/lib/entities/index.ts index c780e05..14095a5 100644 --- a/libs/database/src/lib/entities/index.ts +++ b/libs/database/src/lib/entities/index.ts @@ -18,4 +18,5 @@ export * from "./project-user.entity"; export * from "./role.entity"; export * from "./site.entity"; export * from "./site-polygon.entity"; +export * from "./tree-species.entity"; export * from "./user.entity"; diff --git a/libs/database/src/lib/entities/site.entity.ts b/libs/database/src/lib/entities/site.entity.ts index 0b96383..bd05825 100644 --- a/libs/database/src/lib/entities/site.entity.ts +++ b/libs/database/src/lib/entities/site.entity.ts @@ -1,9 +1,10 @@ -import { AutoIncrement, Column, Index, Model, PrimaryKey, Table } from "sequelize-typescript"; +import { AutoIncrement, Column, HasMany, Index, Model, PrimaryKey, Table } from "sequelize-typescript"; import { BIGINT, UUID } from "sequelize"; +import { TreeSpecies } from "./tree-species.entity"; // A quick stub for the research endpoints @Table({ tableName: "v2_sites", underscored: true, paranoid: true }) -export class Site extends Model { +export class Site extends Model { @PrimaryKey @AutoIncrement @Column(BIGINT.UNSIGNED) @@ -12,4 +13,17 @@ export class Site extends Model { @Index @Column(UUID) uuid: string; + + @HasMany(() => TreeSpecies, { + foreignKey: "speciesableId", + scope: { speciesableType: "App\\Models\\V2\\Sites\\Site" } + }) + treeSpecies: TreeSpecies[] | null; + + async loadTreeSpecies() { + if (this.treeSpecies == null) { + this.treeSpecies = await this.$get("treeSpecies"); + } + return this.treeSpecies; + } } diff --git a/libs/database/src/lib/entities/tree-species.entity.ts b/libs/database/src/lib/entities/tree-species.entity.ts new file mode 100644 index 0000000..d7bf87f --- /dev/null +++ b/libs/database/src/lib/entities/tree-species.entity.ts @@ -0,0 +1,55 @@ +import { + AllowNull, + AutoIncrement, + BelongsTo, + Column, + Index, + Model, + PrimaryKey, + Table, + Unique +} from "sequelize-typescript"; +import { BIGINT, BOOLEAN, STRING, UUID } from "sequelize"; +import { Site } from "./site.entity"; + +@Table({ tableName: "v2_tree_species", underscored: true, paranoid: true }) +export class TreeSpecies extends Model { + @PrimaryKey + @AutoIncrement + @Column(BIGINT.UNSIGNED) + override id: number; + + @Unique + @Column(UUID) + uuid: string; + + @AllowNull + @Column(STRING) + name: string | null; + + @AllowNull + @Column(BIGINT) + amount: number | null; + + @AllowNull + @Index("v2_tree_species_collection_index") + @Index("tree_species_type_id_collection") + @Column(STRING) + collection: string | null; + + @Column({ type: BOOLEAN, defaultValue: false }) + hidden: boolean; + + @Column(STRING) + @Index("tree_species_type_id_collection") + @Index("v2_tree_species_morph_index") + speciesableType: string; + + @Column(BIGINT.UNSIGNED) + @Index("tree_species_type_id_collection") + @Index("v2_tree_species_morph_index") + speciesableId: number; + + @BelongsTo(() => Site, { foreignKey: "speciesableId", scope: { speciesableType: "App\\Models\\V2\\Sites\\Site" } }) + site: Site | null; +} From 7c494d9a837a9746154f3a420b2cb146c9bca5d2 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Thu, 7 Nov 2024 15:54:03 -0800 Subject: [PATCH 21/41] [TM-1451] Implement reporting periods. --- .../src/site-polygons/dto/site-polygon.dto.ts | 15 ++++-- .../site-polygons/site-polygons.controller.ts | 6 ++- .../site-polygons/site-polygons.service.ts | 25 +++++++-- libs/database/src/lib/entities/index.ts | 1 + .../src/lib/entities/site-report.entity.ts | 52 +++++++++++++++++++ libs/database/src/lib/entities/site.entity.ts | 11 ++++ .../src/lib/entities/tree-species.entity.ts | 7 +++ 7 files changed, 108 insertions(+), 9 deletions(-) create mode 100644 libs/database/src/lib/entities/site-report.entity.ts diff --git a/apps/research-service/src/site-polygons/dto/site-polygon.dto.ts b/apps/research-service/src/site-polygons/dto/site-polygon.dto.ts index dd15c7e..22d9f23 100644 --- a/apps/research-service/src/site-polygons/dto/site-polygon.dto.ts +++ b/apps/research-service/src/site-polygons/dto/site-polygon.dto.ts @@ -28,7 +28,7 @@ export class TreeSpeciesDto { amount: number | null; } -class ReportingPeriod { +export class ReportingPeriodDto { @ApiProperty() dueAt: Date; @@ -45,14 +45,19 @@ class ReportingPeriod { @JsonApiDto({ type: "sitePolygons" }) export class SitePolygonDto extends JsonApiAttributes { - constructor(sitePolygon: SitePolygon, indicators: IndicatorDto[], establishmentTreeSpecies: TreeSpeciesDto[]) { + constructor( + sitePolygon: SitePolygon, + indicators: IndicatorDto[], + establishmentTreeSpecies: TreeSpeciesDto[], + reportingPeriods: ReportingPeriodDto[] + ) { super({ ...pickApiProperties(sitePolygon, SitePolygonDto), name: sitePolygon.polyName, siteId: sitePolygon.siteUuid, indicators, establishmentTreeSpecies, - reportingPeriods: [] + reportingPeriods }); } @@ -110,9 +115,9 @@ export class SitePolygonDto extends JsonApiAttributes { establishmentTreeSpecies: TreeSpeciesDto[]; @ApiProperty({ - type: () => ReportingPeriod, + type: () => ReportingPeriodDto, isArray: true, description: "Access to reported trees planted for each approved report on this site." }) - reportingPeriods: ReportingPeriod[]; + reportingPeriods: ReportingPeriodDto[]; } diff --git a/apps/research-service/src/site-polygons/site-polygons.controller.ts b/apps/research-service/src/site-polygons/site-polygons.controller.ts index dd10ebe..ccb415b 100644 --- a/apps/research-service/src/site-polygons/site-polygons.controller.ts +++ b/apps/research-service/src/site-polygons/site-polygons.controller.ts @@ -56,7 +56,11 @@ export class SitePolygonsController { for (const sitePolygon of await builder.execute()) { const indicators = await this.sitePolygonService.convertIndicators(sitePolygon); const establishmentTreeSpecies = await this.sitePolygonService.getEstablishmentTreeSpecies(sitePolygon); - document.addData(sitePolygon.uuid, new SitePolygonDto(sitePolygon, indicators, establishmentTreeSpecies)); + const reportingPeriods = await this.sitePolygonService.getReportingPeriods(sitePolygon); + document.addData( + sitePolygon.uuid, + new SitePolygonDto(sitePolygon, indicators, establishmentTreeSpecies, reportingPeriods) + ); } return document.serialize(); } diff --git a/apps/research-service/src/site-polygons/site-polygons.service.ts b/apps/research-service/src/site-polygons/site-polygons.service.ts index 6a68bcb..6ab80ae 100644 --- a/apps/research-service/src/site-polygons/site-polygons.service.ts +++ b/apps/research-service/src/site-polygons/site-polygons.service.ts @@ -1,7 +1,7 @@ import { BadRequestException, Injectable, Type } from "@nestjs/common"; -import { Site, SitePolygon } from "@terramatch-microservices/database/entities"; +import { Site, SitePolygon, SiteReport } from "@terramatch-microservices/database/entities"; import { Attributes, FindOptions, Op, WhereOptions } from "sequelize"; -import { IndicatorDto, TreeSpeciesDto } from "./dto/site-polygon.dto"; +import { IndicatorDto, ReportingPeriodDto, TreeSpeciesDto } from "./dto/site-polygon.dto"; import { INDICATOR_DTOS } from "./dto/indicators.dto"; import { ModelPropertiesAccessor } from "@nestjs/swagger/dist/services/model-properties-accessor"; import { pick } from "lodash"; @@ -15,7 +15,10 @@ class SitePolygonQueryBuilder { "indicatorsTreeCount", "indicatorsTreeCover", "indicatorsTreeCoverLoss", - { model: Site, include: ["treeSpecies"] } + { + model: Site, + include: ["treeSpecies", { model: SiteReport, include: ["treeSpecies"] }] + } ] }; @@ -66,4 +69,20 @@ export class SitePolygonsService { const site = await sitePolygon.loadSite(); return (await site.loadTreeSpecies()).map(({ name, amount }) => ({ name, amount })); } + + async getReportingPeriods(sitePolygon: SitePolygon): Promise { + // These associations are expected to be eager loaded, so this should not result in new SQL + // queries + const site = await sitePolygon.loadSite(); + const reportingPeriods: ReportingPeriodDto[] = []; + for (const report of await site.loadSiteReports()) { + reportingPeriods.push({ + dueAt: report.dueAt, + submittedAt: report.submittedAt, + treeSpecies: (await report.loadTreeSpecies()).map(({ name, amount }) => ({ name, amount })) + }); + } + + return reportingPeriods; + } } diff --git a/libs/database/src/lib/entities/index.ts b/libs/database/src/lib/entities/index.ts index 14095a5..8ed74c5 100644 --- a/libs/database/src/lib/entities/index.ts +++ b/libs/database/src/lib/entities/index.ts @@ -18,5 +18,6 @@ export * from "./project-user.entity"; export * from "./role.entity"; export * from "./site.entity"; export * from "./site-polygon.entity"; +export * from "./site-report.entity"; export * from "./tree-species.entity"; export * from "./user.entity"; diff --git a/libs/database/src/lib/entities/site-report.entity.ts b/libs/database/src/lib/entities/site-report.entity.ts new file mode 100644 index 0000000..cba5fbe --- /dev/null +++ b/libs/database/src/lib/entities/site-report.entity.ts @@ -0,0 +1,52 @@ +import { + AllowNull, + AutoIncrement, + Column, + ForeignKey, + HasMany, + Index, + Model, + PrimaryKey, + Table +} from "sequelize-typescript"; +import { BIGINT, DATE, UUID } from "sequelize"; +import { TreeSpecies } from "./tree-species.entity"; +import { Site } from "./site.entity"; + +// A quick stub for the research endpoints +@Table({ tableName: "v2_site_reports", underscored: true, paranoid: true }) +export class SiteReport extends Model { + @PrimaryKey + @AutoIncrement + @Column(BIGINT.UNSIGNED) + override id: number; + + @Index + @Column(UUID) + uuid: string; + + @ForeignKey(() => Site) + @Column(BIGINT.UNSIGNED) + siteId: number; + + @AllowNull + @Column(DATE) + dueAt: Date | null; + + @AllowNull + @Column(DATE) + submittedAt: Date | null; + + @HasMany(() => TreeSpecies, { + foreignKey: "speciesableId", + scope: { speciesableType: "App\\Models\\V2\\Sites\\SiteReport" } + }) + treeSpecies: TreeSpecies[] | null; + + async loadTreeSpecies() { + if (this.treeSpecies == null) { + this.treeSpecies = await this.$get("treeSpecies"); + } + return this.treeSpecies; + } +} diff --git a/libs/database/src/lib/entities/site.entity.ts b/libs/database/src/lib/entities/site.entity.ts index bd05825..8ec2dad 100644 --- a/libs/database/src/lib/entities/site.entity.ts +++ b/libs/database/src/lib/entities/site.entity.ts @@ -1,6 +1,7 @@ import { AutoIncrement, Column, HasMany, Index, Model, PrimaryKey, Table } from "sequelize-typescript"; import { BIGINT, UUID } from "sequelize"; import { TreeSpecies } from "./tree-species.entity"; +import { SiteReport } from "./site-report.entity"; // A quick stub for the research endpoints @Table({ tableName: "v2_sites", underscored: true, paranoid: true }) @@ -26,4 +27,14 @@ export class Site extends Model { } return this.treeSpecies; } + + @HasMany(() => SiteReport) + siteReports: SiteReport[] | null; + + async loadSiteReports() { + if (this.siteReports == null) { + this.siteReports = await this.$get("siteReports"); + } + return this.siteReports; + } } diff --git a/libs/database/src/lib/entities/tree-species.entity.ts b/libs/database/src/lib/entities/tree-species.entity.ts index d7bf87f..8775fab 100644 --- a/libs/database/src/lib/entities/tree-species.entity.ts +++ b/libs/database/src/lib/entities/tree-species.entity.ts @@ -11,6 +11,7 @@ import { } from "sequelize-typescript"; import { BIGINT, BOOLEAN, STRING, UUID } from "sequelize"; import { Site } from "./site.entity"; +import { SiteReport } from "./site-report.entity"; @Table({ tableName: "v2_tree_species", underscored: true, paranoid: true }) export class TreeSpecies extends Model { @@ -52,4 +53,10 @@ export class TreeSpecies extends Model { @BelongsTo(() => Site, { foreignKey: "speciesableId", scope: { speciesableType: "App\\Models\\V2\\Sites\\Site" } }) site: Site | null; + + @BelongsTo(() => SiteReport, { + foreignKey: "speciesableId", + scope: { speciesableType: "App\\Models\\V2\\Sites\\SiteReport" } + }) + siteReport: SiteReport | null; } From f71127f1eab1f203b4e04c31db1ddfd7a67c9421 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Thu, 7 Nov 2024 15:59:18 -0800 Subject: [PATCH 22/41] [TM-1451] Be resilient to a missing site. --- .../src/site-polygons/site-polygons.service.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apps/research-service/src/site-polygons/site-polygons.service.ts b/apps/research-service/src/site-polygons/site-polygons.service.ts index 6ab80ae..ffe95d3 100644 --- a/apps/research-service/src/site-polygons/site-polygons.service.ts +++ b/apps/research-service/src/site-polygons/site-polygons.service.ts @@ -67,6 +67,8 @@ export class SitePolygonsService { // These associations are expected to be eager loaded, so this should not result in new SQL // queries. const site = await sitePolygon.loadSite(); + if (site == null) return []; + return (await site.loadTreeSpecies()).map(({ name, amount }) => ({ name, amount })); } @@ -74,6 +76,8 @@ export class SitePolygonsService { // These associations are expected to be eager loaded, so this should not result in new SQL // queries const site = await sitePolygon.loadSite(); + if (site == null) return []; + const reportingPeriods: ReportingPeriodDto[] = []; for (const report of await site.loadSiteReports()) { reportingPeriods.push({ From 45e298e9f72a0aa86b6e5092b40883daca989f20 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Thu, 7 Nov 2024 16:08:17 -0800 Subject: [PATCH 23/41] [TM-1451] Include polygon geometry. --- .../src/site-polygons/dto/site-polygon.dto.ts | 10 +++++++++- .../src/site-polygons/site-polygons.controller.ts | 3 ++- .../src/site-polygons/site-polygons.service.ts | 1 + 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/apps/research-service/src/site-polygons/dto/site-polygon.dto.ts b/apps/research-service/src/site-polygons/dto/site-polygon.dto.ts index 22d9f23..b4fab58 100644 --- a/apps/research-service/src/site-polygons/dto/site-polygon.dto.ts +++ b/apps/research-service/src/site-polygons/dto/site-polygon.dto.ts @@ -11,6 +11,7 @@ import { } from "./indicators.dto"; import { POLYGON_STATUSES, PolygonStatus } from "@terramatch-microservices/database/constants"; import { SitePolygon } from "@terramatch-microservices/database/entities"; +import { Polygon } from "geojson"; export type IndicatorDto = | IndicatorTreeCoverLossDto @@ -47,6 +48,7 @@ export class ReportingPeriodDto { export class SitePolygonDto extends JsonApiAttributes { constructor( sitePolygon: SitePolygon, + geometry: Polygon, indicators: IndicatorDto[], establishmentTreeSpecies: TreeSpeciesDto[], reportingPeriods: ReportingPeriodDto[] @@ -55,6 +57,7 @@ export class SitePolygonDto extends JsonApiAttributes { ...pickApiProperties(sitePolygon, SitePolygonDto), name: sitePolygon.polyName, siteId: sitePolygon.siteUuid, + geometry, indicators, establishmentTreeSpecies, reportingPeriods @@ -67,9 +70,14 @@ export class SitePolygonDto extends JsonApiAttributes { @ApiProperty({ enum: POLYGON_STATUSES }) status: PolygonStatus; - @ApiProperty() + @ApiProperty({ + description: "If this ID points to a deleted site, the tree species and reporting period will be empty." + }) siteId: string; + @ApiProperty() + geometry: Polygon; + @ApiProperty({ nullable: true }) plantStart: Date | null; diff --git a/apps/research-service/src/site-polygons/site-polygons.controller.ts b/apps/research-service/src/site-polygons/site-polygons.controller.ts index ccb415b..7a3ee14 100644 --- a/apps/research-service/src/site-polygons/site-polygons.controller.ts +++ b/apps/research-service/src/site-polygons/site-polygons.controller.ts @@ -54,12 +54,13 @@ export class SitePolygonsController { const document = buildJsonApi(); for (const sitePolygon of await builder.execute()) { + const geometry = await sitePolygon.loadPolygon(); const indicators = await this.sitePolygonService.convertIndicators(sitePolygon); const establishmentTreeSpecies = await this.sitePolygonService.getEstablishmentTreeSpecies(sitePolygon); const reportingPeriods = await this.sitePolygonService.getReportingPeriods(sitePolygon); document.addData( sitePolygon.uuid, - new SitePolygonDto(sitePolygon, indicators, establishmentTreeSpecies, reportingPeriods) + new SitePolygonDto(sitePolygon, geometry?.polygon, indicators, establishmentTreeSpecies, reportingPeriods) ); } return document.serialize(); diff --git a/apps/research-service/src/site-polygons/site-polygons.service.ts b/apps/research-service/src/site-polygons/site-polygons.service.ts index ffe95d3..c98617c 100644 --- a/apps/research-service/src/site-polygons/site-polygons.service.ts +++ b/apps/research-service/src/site-polygons/site-polygons.service.ts @@ -15,6 +15,7 @@ class SitePolygonQueryBuilder { "indicatorsTreeCount", "indicatorsTreeCover", "indicatorsTreeCoverLoss", + "polygon", { model: Site, include: ["treeSpecies", { model: SiteReport, include: ["treeSpecies"] }] From dcee5d5d97a1ccad4091a6e8e3faefb82bc4d714 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Thu, 7 Nov 2024 16:16:29 -0800 Subject: [PATCH 24/41] [TM-1451] Rename a few things for consistency and readability. --- .../src/site-polygons/site-polygons.controller.ts | 12 ++++++------ .../src/site-polygons/site-polygons.service.ts | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/apps/research-service/src/site-polygons/site-polygons.controller.ts b/apps/research-service/src/site-polygons/site-polygons.controller.ts index 7a3ee14..74dc1a5 100644 --- a/apps/research-service/src/site-polygons/site-polygons.controller.ts +++ b/apps/research-service/src/site-polygons/site-polygons.controller.ts @@ -25,7 +25,7 @@ import { import { SitePolygonBulkUpdateBodyDto } from "./dto/site-polygon-update.dto"; import { SitePolygonsService } from "./site-polygons.service"; -const DEFAULT_PAGE_SIZE = 100 as const; +const MAX_PAGE_SIZE = 100 as const; @Controller("research/v3/sitePolygons") @ApiExtraModels( @@ -45,17 +45,17 @@ export class SitePolygonsController { @ApiException(() => UnauthorizedException, { description: "Authentication failed." }) @ApiException(() => BadRequestException, { description: "Pagination values are invalid." }) async findMany(@Query() query?: SitePolygonQueryDto): Promise { - const { size: pageSize = DEFAULT_PAGE_SIZE, after: pageAfter } = query.page ?? {}; - if (pageSize > DEFAULT_PAGE_SIZE || pageSize < 1) { + const { size: pageSize = MAX_PAGE_SIZE, after: pageAfter } = query.page ?? {}; + if (pageSize > MAX_PAGE_SIZE || pageSize < 1) { throw new BadRequestException("Page size is invalid"); } - const builder = await this.sitePolygonService.buildQuery(pageSize, pageAfter); + const queryBuilder = await this.sitePolygonService.buildQuery(pageSize, pageAfter); const document = buildJsonApi(); - for (const sitePolygon of await builder.execute()) { + for (const sitePolygon of await queryBuilder.execute()) { const geometry = await sitePolygon.loadPolygon(); - const indicators = await this.sitePolygonService.convertIndicators(sitePolygon); + const indicators = await this.sitePolygonService.getIndicators(sitePolygon); const establishmentTreeSpecies = await this.sitePolygonService.getEstablishmentTreeSpecies(sitePolygon); const reportingPeriods = await this.sitePolygonService.getReportingPeriods(sitePolygon); document.addData( diff --git a/apps/research-service/src/site-polygons/site-polygons.service.ts b/apps/research-service/src/site-polygons/site-polygons.service.ts index c98617c..bbf6249 100644 --- a/apps/research-service/src/site-polygons/site-polygons.service.ts +++ b/apps/research-service/src/site-polygons/site-polygons.service.ts @@ -52,7 +52,7 @@ export class SitePolygonsService { return builder; } - async convertIndicators(sitePolygon: SitePolygon): Promise { + async getIndicators(sitePolygon: SitePolygon): Promise { const accessor = new ModelPropertiesAccessor(); const indicators: IndicatorDto[] = []; for (const indicator of await sitePolygon.getIndicators()) { From 82a3ed42a37bedc935c7c5dc52391d9d878e1d7a Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Thu, 7 Nov 2024 21:06:21 -0800 Subject: [PATCH 25/41] [TM-1451] Set not required on all query params. --- .../site-polygons/dto/site-polygon-query.dto.ts | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/apps/research-service/src/site-polygons/dto/site-polygon-query.dto.ts b/apps/research-service/src/site-polygons/dto/site-polygon-query.dto.ts index 0d156b2..62da244 100644 --- a/apps/research-service/src/site-polygons/dto/site-polygon-query.dto.ts +++ b/apps/research-service/src/site-polygons/dto/site-polygon-query.dto.ts @@ -13,7 +13,8 @@ class Page { description: "The size of page being requested", minimum: 1, maximum: 100, - default: 100 + default: 100, + required: false }) @IsOptional() @IsInt() @@ -21,6 +22,7 @@ class Page { @ApiProperty({ name: "page[after]", + required: false, description: "The last record before the page being requested. The value is a polygon UUID. If not provided, the first page is returned." }) @@ -33,6 +35,7 @@ export class SitePolygonQueryDto { enum: POLYGON_STATUSES, name: "polygonStatus[]", isArray: true, + required: false, description: "Filter results by polygon status" }) @IsOptional() @@ -42,6 +45,7 @@ export class SitePolygonQueryDto { @ApiProperty({ name: "projectId[]", isArray: true, + required: false, description: "Filter results by project UUID(s)" }) @IsOptional() @@ -52,24 +56,29 @@ export class SitePolygonQueryDto { enum: INDICATOR_SLUGS, name: "missingIndicator[]", isArray: true, + required: false, description: "Filter results by polygons that are missing at least one of the indicators listed" }) @IsOptional() @IsArray() missingIndicator?: IndicatorSlug[]; - @ApiProperty({ description: "Filter results by polygons that have been modified since the date provided" }) + @ApiProperty({ + required: false, + description: "Filter results by polygons that have been modified since the date provided" + }) @IsOptional() @IsDate() lastModifiedDate?: Date; @ApiProperty({ + required: false, description: "Filter results by polygons that are within the boundary of the polygon referenced by this UUID" }) @IsOptional() boundaryPolygon?: string; - @ApiProperty({ name: "page", description: "Pagination information" }) + @ApiProperty({ name: "page", required: false, description: "Pagination information" }) @ValidateNested() @IsOptional() page?: Page; From c2bbacfa82d4c02f6104761a9beb8691beaefd5b Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Thu, 7 Nov 2024 21:18:37 -0800 Subject: [PATCH 26/41] [TM-1451] Implement API key authorization. --- libs/common/src/lib/guards/auth.guard.ts | 44 ++++++++++++++---------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/libs/common/src/lib/guards/auth.guard.ts b/libs/common/src/lib/guards/auth.guard.ts index f60ef7f..6876cd0 100644 --- a/libs/common/src/lib/guards/auth.guard.ts +++ b/libs/common/src/lib/guards/auth.guard.ts @@ -1,13 +1,9 @@ -import { - CanActivate, - ExecutionContext, - Injectable, SetMetadata, - UnauthorizedException -} from '@nestjs/common'; -import { JwtService } from '@nestjs/jwt'; -import { Reflector } from '@nestjs/core'; - -const NO_BEARER_AUTH = 'noBearerAuth'; +import { CanActivate, ExecutionContext, Injectable, SetMetadata, UnauthorizedException } from "@nestjs/common"; +import { JwtService } from "@nestjs/jwt"; +import { Reflector } from "@nestjs/core"; +import { User } from "@terramatch-microservices/database/entities"; + +const NO_BEARER_AUTH = "noBearerAuth"; export const NoBearerAuth = SetMetadata(NO_BEARER_AUTH, true); @Injectable() @@ -17,22 +13,34 @@ export class AuthGuard implements CanActivate { async canActivate(context: ExecutionContext): Promise { const skipAuth = this.reflector.getAllAndOverride(NO_BEARER_AUTH, [ context.getHandler(), - context.getClass(), + context.getClass() ]); if (skipAuth) return true; const request = context.switchToHttp().getRequest(); - const [type, token] = request.headers.authorization?.split(' ') ?? []; - if (type !== 'Bearer' || token == null) throw new UnauthorizedException(); + const [type, token] = request.headers.authorization?.split(" ") ?? []; + if (type !== "Bearer" || token == null) throw new UnauthorizedException(); + + const userId = (await this.getJwtUserId(token)) ?? (await this.getApiKeyUserId(token)); + if (userId == null) throw new UnauthorizedException(); + + // Most requests won't need the actual user object; instead the roles and permissions + // are fetched from other (smaller) tables, and only the user id is needed. + request.authenticatedUserId = userId; + return true; + } + private async getJwtUserId(token: string) { try { const { sub } = await this.jwtService.verifyAsync(token); - // Most requests won't need the actual user object; instead the roles and permissions - // are fetched from other (smaller) tables, and only the user id is needed. - request.authenticatedUserId = sub; - return true; + return sub; } catch { - throw new UnauthorizedException(); + return null; } } + + private async getApiKeyUserId(token: string) { + const { id } = (await User.findOne({ where: { apiKey: token }, attributes: ["id"] })) ?? {}; + return id; + } } From 7c7bd50d43ca332bdc6e5a2cacc6f69d2f62ce28 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Thu, 7 Nov 2024 21:44:08 -0800 Subject: [PATCH 27/41] [TM-1451] Implement site polygon policy. --- .../site-polygons/site-polygons.controller.ts | 9 ++++- .../common/src/lib/policies/policy.service.ts | 35 +++++++++---------- .../src/lib/policies/site-polygon.policy.ts | 10 ++++++ 3 files changed, 35 insertions(+), 19 deletions(-) create mode 100644 libs/common/src/lib/policies/site-polygon.policy.ts diff --git a/apps/research-service/src/site-polygons/site-polygons.controller.ts b/apps/research-service/src/site-polygons/site-polygons.controller.ts index 74dc1a5..d0f2d1e 100644 --- a/apps/research-service/src/site-polygons/site-polygons.controller.ts +++ b/apps/research-service/src/site-polygons/site-polygons.controller.ts @@ -24,6 +24,8 @@ import { } from "./dto/indicators.dto"; import { SitePolygonBulkUpdateBodyDto } from "./dto/site-polygon-update.dto"; import { SitePolygonsService } from "./site-polygons.service"; +import { PolicyService } from "@terramatch-microservices/common"; +import { SitePolygon } from "@terramatch-microservices/database/entities"; const MAX_PAGE_SIZE = 100 as const; @@ -37,7 +39,10 @@ const MAX_PAGE_SIZE = 100 as const; IndicatorMsuCarbonDto ) export class SitePolygonsController { - constructor(private readonly sitePolygonService: SitePolygonsService) {} + constructor( + private readonly sitePolygonService: SitePolygonsService, + private readonly policyService: PolicyService + ) {} @Get() @ApiOperation({ operationId: "sitePolygonsIndex", summary: "Get all site polygons" }) @@ -45,6 +50,8 @@ export class SitePolygonsController { @ApiException(() => UnauthorizedException, { description: "Authentication failed." }) @ApiException(() => BadRequestException, { description: "Pagination values are invalid." }) async findMany(@Query() query?: SitePolygonQueryDto): Promise { + await this.policyService.authorize("readAll", SitePolygon); + const { size: pageSize = MAX_PAGE_SIZE, after: pageAfter } = query.page ?? {}; if (pageSize > MAX_PAGE_SIZE || pageSize < 1) { throw new BadRequestException("Page size is invalid"); diff --git a/libs/common/src/lib/policies/policy.service.ts b/libs/common/src/lib/policies/policy.service.ts index 4b67505..c8e5bdd 100644 --- a/libs/common/src/lib/policies/policy.service.ts +++ b/libs/common/src/lib/policies/policy.service.ts @@ -1,15 +1,12 @@ -import { - Injectable, - LoggerService, - UnauthorizedException, -} from '@nestjs/common'; -import { RequestContext } from 'nestjs-request-context'; -import { UserPolicy } from './user.policy'; -import { BuilderType, EntityPolicy } from './entity.policy'; -import { Permission, User } from '@terramatch-microservices/database/entities'; -import { AbilityBuilder, createMongoAbility } from '@casl/ability'; -import { Model } from 'sequelize-typescript'; -import { TMLogService } from '../util/tm-log.service'; +import { Injectable, LoggerService, UnauthorizedException } from "@nestjs/common"; +import { RequestContext } from "nestjs-request-context"; +import { UserPolicy } from "./user.policy"; +import { BuilderType, EntityPolicy } from "./entity.policy"; +import { Permission, SitePolygon, User } from "@terramatch-microservices/database/entities"; +import { AbilityBuilder, createMongoAbility } from "@casl/ability"; +import { Model } from "sequelize-typescript"; +import { TMLogService } from "../util/tm-log.service"; +import { SitePolygonPolicy } from "./site-polygon.policy"; type EntityClass = { // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -19,10 +16,11 @@ type EntityClass = { type PolicyClass = { new (userId: number, permissions: string[], builder: AbilityBuilder): EntityPolicy; -} +}; -const POLICIES: [ [EntityClass, PolicyClass] ] = [ - [User, UserPolicy] +const POLICIES: [EntityClass, PolicyClass][] = [ + [User, UserPolicy], + [SitePolygon, SitePolygonPolicy] ]; /** @@ -39,12 +37,13 @@ const POLICIES: [ [EntityClass, PolicyClass] ] = [ export class PolicyService { private readonly log: LoggerService = new TMLogService(PolicyService.name); - async authorize(action: string, subject: T): Promise { + async authorize(action: string, subject: Model | EntityClass): Promise { // Added by AuthGuard const userId = RequestContext.currentContext.req.authenticatedUserId; if (userId == null) throw new UnauthorizedException(); - const [, PolicyClass] = POLICIES.find(([entityClass]) => subject instanceof entityClass) ?? []; + const [, PolicyClass] = + POLICIES.find(([entityClass]) => subject instanceof entityClass || subject === entityClass) ?? []; if (PolicyClass == null) { this.log.error(`No policy found for subject type [${subject.constructor.name}]`); throw new UnauthorizedException(); @@ -52,7 +51,7 @@ export class PolicyService { const permissions = await Permission.getUserPermissionNames(userId); const builder = new AbilityBuilder(createMongoAbility); - await (new PolicyClass(userId, permissions, builder)).addRules(); + await new PolicyClass(userId, permissions, builder).addRules(); const ability = builder.build(); if (!ability.can(action, subject)) throw new UnauthorizedException(); diff --git a/libs/common/src/lib/policies/site-polygon.policy.ts b/libs/common/src/lib/policies/site-polygon.policy.ts new file mode 100644 index 0000000..727071d --- /dev/null +++ b/libs/common/src/lib/policies/site-polygon.policy.ts @@ -0,0 +1,10 @@ +import { EntityPolicy } from "./entity.policy"; +import { SitePolygon } from "@terramatch-microservices/database/entities"; + +export class SitePolygonPolicy extends EntityPolicy { + async addRules() { + if (this.permissions.includes("polygons-manage")) { + this.builder.can("manage", SitePolygon); + } + } +} From 7771803a2ae9693d2d364da76cfc2b9e877e60d7 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Thu, 7 Nov 2024 21:48:47 -0800 Subject: [PATCH 28/41] [TM-1451] Be a little smarter about when we check for an api key. --- libs/common/src/lib/guards/auth.guard.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/libs/common/src/lib/guards/auth.guard.ts b/libs/common/src/lib/guards/auth.guard.ts index 6876cd0..5ad5dad 100644 --- a/libs/common/src/lib/guards/auth.guard.ts +++ b/libs/common/src/lib/guards/auth.guard.ts @@ -21,7 +21,7 @@ export class AuthGuard implements CanActivate { const [type, token] = request.headers.authorization?.split(" ") ?? []; if (type !== "Bearer" || token == null) throw new UnauthorizedException(); - const userId = (await this.getJwtUserId(token)) ?? (await this.getApiKeyUserId(token)); + const userId = this.isJwtToken(token) ? await this.getJwtUserId(token) : await this.getApiKeyUserId(token); if (userId == null) throw new UnauthorizedException(); // Most requests won't need the actual user object; instead the roles and permissions @@ -30,6 +30,10 @@ export class AuthGuard implements CanActivate { return true; } + private isJwtToken(token: string) { + return this.jwtService.decode(token) != null; + } + private async getJwtUserId(token: string) { try { const { sub } = await this.jwtService.verifyAsync(token); From d81afa150c43835500c243b7afdb8dd95ad29072 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Thu, 7 Nov 2024 22:15:19 -0800 Subject: [PATCH 29/41] [TM-1451] Include pagination details in API response. --- .../site-polygons/site-polygons.controller.ts | 5 +- .../decorators/json-api-response.decorator.ts | 125 ++++++++++-------- libs/common/src/lib/util/json-api-builder.ts | 41 +++++- 3 files changed, 111 insertions(+), 60 deletions(-) diff --git a/apps/research-service/src/site-polygons/site-polygons.controller.ts b/apps/research-service/src/site-polygons/site-polygons.controller.ts index d0f2d1e..aac96c8 100644 --- a/apps/research-service/src/site-polygons/site-polygons.controller.ts +++ b/apps/research-service/src/site-polygons/site-polygons.controller.ts @@ -46,7 +46,7 @@ export class SitePolygonsController { @Get() @ApiOperation({ operationId: "sitePolygonsIndex", summary: "Get all site polygons" }) - @JsonApiResponse({ data: { type: SitePolygonDto }, hasMany: true, pagination: true }) + @JsonApiResponse({ data: { type: SitePolygonDto }, pagination: true }) @ApiException(() => UnauthorizedException, { description: "Authentication failed." }) @ApiException(() => BadRequestException, { description: "Pagination values are invalid." }) async findMany(@Query() query?: SitePolygonQueryDto): Promise { @@ -59,7 +59,7 @@ export class SitePolygonsController { const queryBuilder = await this.sitePolygonService.buildQuery(pageSize, pageAfter); - const document = buildJsonApi(); + const document = buildJsonApi({ pagination: true }); for (const sitePolygon of await queryBuilder.execute()) { const geometry = await sitePolygon.loadPolygon(); const indicators = await this.sitePolygonService.getIndicators(sitePolygon); @@ -70,6 +70,7 @@ export class SitePolygonsController { new SitePolygonDto(sitePolygon, geometry?.polygon, indicators, establishmentTreeSpecies, reportingPeriods) ); } + return document.serialize(); } diff --git a/libs/common/src/lib/decorators/json-api-response.decorator.ts b/libs/common/src/lib/decorators/json-api-response.decorator.ts index dc2ba53..4288a72 100644 --- a/libs/common/src/lib/decorators/json-api-response.decorator.ts +++ b/libs/common/src/lib/decorators/json-api-response.decorator.ts @@ -1,42 +1,43 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ -import { ApiExtraModels, ApiResponse, ApiResponseOptions, getSchemaPath } from '@nestjs/swagger'; -import { applyDecorators, HttpStatus } from '@nestjs/common'; -import { DTO_ID_METADATA, DTO_TYPE_METADATA, IdType } from './json-api-dto.decorator'; -import { JsonApiAttributes } from '../dto/json-api-attributes'; +import { ApiExtraModels, ApiResponse, ApiResponseOptions, getSchemaPath } from "@nestjs/swagger"; +import { applyDecorators, HttpStatus } from "@nestjs/common"; +import { DTO_ID_METADATA, DTO_TYPE_METADATA, IdType } from "./json-api-dto.decorator"; +import { JsonApiAttributes } from "../dto/json-api-attributes"; type TypeProperties = { - type: 'string'; + type: "string"; example: string; -} +}; type IdProperties = { - type: 'string'; + type: "string"; format?: string; pattern?: string; -} +}; type ResourceDef = { type: TypeProperties; id: IdProperties; attributes: object; relationships?: { - type: 'object'; + type: "object"; properties: { - [key: string]: object - } - } -} + [key: string]: object; + }; + }; + meta?: any; +}; function getIdProperties(resourceType: ResourceType): IdProperties { - const id: IdProperties = { type: 'string' }; + const id: IdProperties = { type: "string" }; const idFormat = Reflect.getMetadata(DTO_ID_METADATA, resourceType) as IdType; switch (idFormat) { - case 'uuid': - id.format = 'uuid'; + case "uuid": + id.format = "uuid"; break; - case 'number': - id.pattern = '^\\d{5}$'; + case "number": + id.pattern = "^\\d{5}$"; break; } @@ -44,11 +45,15 @@ function getIdProperties(resourceType: ResourceType): IdProperties { } const getTypeProperties = (resourceType: ResourceType): TypeProperties => ({ - type: 'string', + type: "string", example: Reflect.getMetadata(DTO_TYPE_METADATA, resourceType) }); -function constructResource(resource: Resource) { +type ConstructResourceOptions = { + pagination?: boolean; +}; + +function constructResource(resource: Resource, options?: ConstructResourceOptions) { const def: ResourceDef = { type: getTypeProperties(resource.type), id: getIdProperties(resource.type), @@ -59,34 +64,43 @@ function constructResource(resource: Resource) { }; if (resource.relationships != null && resource.relationships.length > 0) { - def.relationships = { type: 'object', properties: {} }; + def.relationships = { type: "object", properties: {} }; for (const { name, type, multiple, meta } of resource.relationships) { const relationship = { - type: 'object', + type: "object", properties: { type: getTypeProperties(type), - id: getIdProperties(type), + id: getIdProperties(type) } as { [key: string]: any } - } + }; if (meta != null) { - relationship.properties['meta'] = { type: 'object', properties: meta }; + relationship.properties["meta"] = { type: "object", properties: meta }; } if (multiple === true) { - def.relationships.properties[name] = { type: 'array', items: relationship }; + def.relationships.properties[name] = { type: "array", items: relationship }; } else { def.relationships.properties[name] = relationship; } } } + if (options?.pagination) { + addMeta(def, "page", { + type: "object", + properties: { + cursor: { type: "string", description: "The cursor for this record." } + } + }); + } + return def; } -function addMeta (document: Document, name: string, definition: any) { - if (document.meta == null) document.meta = { type: "object", properties: {} }; - document.meta.properties[name] = definition; +function addMeta(def: Document | ResourceDef, name: string, definition: any) { + if (def.meta == null) def.meta = { type: "object", properties: {} }; + def.meta.properties[name] = definition; } type ResourceType = new (...props: any[]) => JsonApiAttributes; @@ -105,9 +119,9 @@ type Relationship = { * If supplied, will fold into the relationship docs. Should be a well-formed OpenAPI definition. */ meta?: { - [key: string]: { [key: string]: any } + [key: string]: { [key: string]: any }; }; -} +}; type Resource = { /** @@ -116,7 +130,7 @@ type Resource = { type: ResourceType; relationships?: Relationship[]; -} +}; type JsonApiResponseProps = { data: Resource; @@ -129,42 +143,42 @@ type JsonApiResponseProps = { /** * Set to true if this endpoint response documentation should include cursor pagination metadata. + * A true value for pagination forces a true value for hasMany. */ pagination?: boolean; included?: Resource[]; -} +}; type Document = { data: any; meta?: any; included?: any; -} +}; /** * Decorator to simplify wrapping the response type from a controller method with the JSON API * response structure. Builds the JSON:API document structure and applies the ApiExtraModels and * ApiResponse decorators. */ -export function JsonApiResponse( - options: ApiResponseOptions & JsonApiResponseProps -) { +export function JsonApiResponse(options: ApiResponseOptions & JsonApiResponseProps) { const { data, hasMany, pagination, included, status, ...rest } = options; const extraModels: ResourceType[] = [data.type]; const document = { - data: hasMany - ? { - type: "array", - items: { - type: "object", - properties: constructResource(data) - } - } - : { - type: "object", - properties: constructResource(data) - } + data: + hasMany || pagination + ? { + type: "array", + items: { + type: "object", + properties: constructResource(data, { pagination }) + } + } + : { + type: "object", + properties: constructResource(data) + } } as Document; if (included != null && included.length > 0) { @@ -176,17 +190,17 @@ export function JsonApiResponse( items: { oneOf: [] } - } + }; } document.included.items.oneOf.push({ type: "object", properties: constructResource(includedResource) - }) + }); } } if (pagination) { - addMeta(document, 'page', { + addMeta(document, "page", { type: "object", properties: { cursor: { type: "string", description: "The cursor for the first record on this page." }, @@ -202,10 +216,7 @@ export function JsonApiResponse( type: "object", properties: document } - } as ApiResponseOptions + } as ApiResponseOptions; - return applyDecorators( - ApiResponse(apiResponseOptions), - ApiExtraModels(...extraModels) - ); + return applyDecorators(ApiResponse(apiResponseOptions), ApiExtraModels(...extraModels)); } diff --git a/libs/common/src/lib/util/json-api-builder.ts b/libs/common/src/lib/util/json-api-builder.ts index 4f607c3..aa20eac 100644 --- a/libs/common/src/lib/util/json-api-builder.ts +++ b/libs/common/src/lib/util/json-api-builder.ts @@ -22,11 +22,26 @@ export type Resource = { id: string; attributes: Attributes; relationships?: Relationships; + meta?: ResourceMeta; +}; + +type DocumentMeta = { + page?: { + cursor?: string; + total: number; + }; +}; + +type ResourceMeta = { + page?: { + cursor: string; + }; }; export type JsonApiDocument = { data: Resource | Resource[]; included?: Resource | Resource[]; + meta?: DocumentMeta; }; export class ResourceBuilder { @@ -77,16 +92,28 @@ export class ResourceBuilder { resource.relationships = this.relationships; } + if (this.documentBuilder.options?.pagination) { + resource.meta = { + page: { cursor: this.id } + }; + } + return resource; } } export class ApiBuilderException extends Error {} +type DocumentBuilderOptions = { + pagination?: boolean; +}; + class DocumentBuilder { data: ResourceBuilder[] = []; included: ResourceBuilder[] = []; + constructor(public readonly options?: DocumentBuilderOptions) {} + addData(id: string, attributes: any): ResourceBuilder { const builder = new ResourceBuilder(id, attributes, this); @@ -129,8 +156,20 @@ class DocumentBuilder { doc.included = this.included.map(resource => resource.serialize()); } + const meta: DocumentMeta = {}; + if (this.options?.pagination) { + meta.page = { + cursor: this.data[0]?.id, + total: this.data.length + }; + } + + if (Object.keys(meta).length > 0) { + doc.meta = meta; + } + return doc; } } -export const buildJsonApi = () => new DocumentBuilder(); +export const buildJsonApi = (options?: DocumentBuilderOptions) => new DocumentBuilder(options); From 21af11ff03f2b04a00488e3549d005559daddeaa Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Fri, 8 Nov 2024 08:32:53 -0800 Subject: [PATCH 30/41] [TM-1451] Double the research service available CPU --- cdk/service-stack/lib/service-stack.ts | 107 +++++++++++++------------ 1 file changed, 55 insertions(+), 52 deletions(-) diff --git a/cdk/service-stack/lib/service-stack.ts b/cdk/service-stack/lib/service-stack.ts index a731ed8..26955fd 100644 --- a/cdk/service-stack/lib/service-stack.ts +++ b/cdk/service-stack/lib/service-stack.ts @@ -1,24 +1,40 @@ -import { Stack, StackProps, Tags } from 'aws-cdk-lib'; -import { PrivateSubnet, SecurityGroup, Vpc } from 'aws-cdk-lib/aws-ec2'; -import { Construct } from 'constructs'; -import { LogGroup } from 'aws-cdk-lib/aws-logs'; -import { Repository } from 'aws-cdk-lib/aws-ecr'; -import { Cluster, ContainerImage, LogDriver } from 'aws-cdk-lib/aws-ecs'; -import { ApplicationLoadBalancedFargateService } from 'aws-cdk-lib/aws-ecs-patterns'; -import { Role } from 'aws-cdk-lib/aws-iam'; -import { upperFirst } from 'lodash'; +import { Stack, StackProps, Tags } from "aws-cdk-lib"; +import { PrivateSubnet, SecurityGroup, Vpc } from "aws-cdk-lib/aws-ec2"; +import { Construct } from "constructs"; +import { LogGroup } from "aws-cdk-lib/aws-logs"; +import { Repository } from "aws-cdk-lib/aws-ecr"; +import { Cluster, ContainerImage, LogDriver } from "aws-cdk-lib/aws-ecs"; +import { + ApplicationLoadBalancedFargateService, + ApplicationLoadBalancedFargateServiceProps +} from "aws-cdk-lib/aws-ecs-patterns"; +import { Role } from "aws-cdk-lib/aws-iam"; +import { upperFirst } from "lodash"; -const extractFromEnv = (...names: string[]) => names.map(name => { - const value = process.env[name]; - if (value == null) throw new Error(`No ${name} defined`) - return value; -}); +const extractFromEnv = (...names: string[]) => + names.map(name => { + const value = process.env[name]; + if (value == null) throw new Error(`No ${name} defined`); + return value; + }); + +type Mutable = { + -readonly [P in keyof T]: T[P]; +}; + +const customizeFargate = (service: string, env: string, props: Mutable) => { + if (service === "research-service") { + props.cpu = 1024; + } + + return props; +}; export class ServiceStack extends Stack { constructor(scope: Construct, id: string, props?: StackProps) { super(scope, id, props); - const [env, service, imageTag] = extractFromEnv('TM_ENV', 'TM_SERVICE', 'IMAGE_TAG'); + const [env, service, imageTag] = extractFromEnv("TM_ENV", "TM_SERVICE", "IMAGE_TAG"); const envName = upperFirst(env); // Identify the most recently updated service docker image @@ -30,40 +46,35 @@ export class ServiceStack extends Stack { const image = ContainerImage.fromEcrRepository(repository, imageTag); // Identify our pre-configured cluster. - const vpc = Vpc.fromLookup(this, 'wri-terramatch-vpc', { - vpcId: 'vpc-0beac5973796d96b1', + const vpc = Vpc.fromLookup(this, "wri-terramatch-vpc", { + vpcId: "vpc-0beac5973796d96b1" + }); + const cluster = Cluster.fromClusterAttributes(this, "terramatch-microservices", { + clusterName: "terramatch-microservices", + clusterArn: "arn:aws:ecs:eu-west-1:603634817705:cluster/terramatch-microservices", + vpc }); - const cluster = Cluster.fromClusterAttributes( - this, - 'terramatch-microservices', - { - clusterName: 'terramatch-microservices', - clusterArn: - 'arn:aws:ecs:eu-west-1:603634817705:cluster/terramatch-microservices', - vpc, - } - ); const securityGroups = [ - SecurityGroup.fromLookupByName(this, 'default', 'default', vpc), - SecurityGroup.fromLookupByName(this, `db-${env}`, `db-${env}`, vpc), + SecurityGroup.fromLookupByName(this, "default", "default", vpc), + SecurityGroup.fromLookupByName(this, `db-${env}`, `db-${env}`, vpc) ]; const privateSubnets = [ - PrivateSubnet.fromPrivateSubnetAttributes(this, 'eu-west-1a', { - subnetId: 'subnet-065992a829eb772a3', - routeTableId: 'rtb-07f85b7827c451bc9', - }), - PrivateSubnet.fromPrivateSubnetAttributes(this, 'eu-west-1b', { - subnetId: 'subnet-0f48d0681051fa49a', - routeTableId: 'rtb-06afefb0f592f11d6', + PrivateSubnet.fromPrivateSubnetAttributes(this, "eu-west-1a", { + subnetId: "subnet-065992a829eb772a3", + routeTableId: "rtb-07f85b7827c451bc9" }), + PrivateSubnet.fromPrivateSubnetAttributes(this, "eu-west-1b", { + subnetId: "subnet-0f48d0681051fa49a", + routeTableId: "rtb-06afefb0f592f11d6" + }) ]; // Create a load-balanced Fargate service and make it public const fargateService = new ApplicationLoadBalancedFargateService( this, `terramatch-${service}-${env}`, - { + customizeFargate(service, env, { serviceName: `terramatch-${service}-${env}`, cluster, cpu: 512, @@ -73,30 +84,22 @@ export class ServiceStack extends Stack { family: `terramatch-${service}-${env}`, containerName: `terramatch-${service}-${env}`, logDriver: LogDriver.awsLogs({ - logGroup: LogGroup.fromLogGroupName( - this, - `${service}-${env}`, - `ecs/${service}-${env}` - ), - streamPrefix: `${service}-${env}`, + logGroup: LogGroup.fromLogGroupName(this, `${service}-${env}`, `ecs/${service}-${env}`), + streamPrefix: `${service}-${env}` }), - executionRole: Role.fromRoleName( - this, - 'ecsTaskExecutionRole', - 'ecsTaskExecutionRole' - ), + executionRole: Role.fromRoleName(this, "ecsTaskExecutionRole", "ecsTaskExecutionRole") }, securityGroups: securityGroups, taskSubnets: { subnets: privateSubnets }, memoryLimitMiB: 2048, assignPublicIp: false, publicLoadBalancer: false, - loadBalancerName: `${service}-${env}`, - } + loadBalancerName: `${service}-${env}` + }) ); fargateService.targetGroup.configureHealthCheck({ - path: '/health', + path: "/health" }); - Tags.of(fargateService.loadBalancer).add('service', `${service}-${env}`); + Tags.of(fargateService.loadBalancer).add("service", `${service}-${env}`); } } From 3c118aed2bebc4cf4fe0a3f29e4873be9eab6d02 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Fri, 8 Nov 2024 08:50:01 -0800 Subject: [PATCH 31/41] [TM-1451] Bump it up a bit more. --- cdk/service-stack/lib/service-stack.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cdk/service-stack/lib/service-stack.ts b/cdk/service-stack/lib/service-stack.ts index 26955fd..293e1bc 100644 --- a/cdk/service-stack/lib/service-stack.ts +++ b/cdk/service-stack/lib/service-stack.ts @@ -24,7 +24,8 @@ type Mutable = { const customizeFargate = (service: string, env: string, props: Mutable) => { if (service === "research-service") { - props.cpu = 1024; + props.cpu = 2048; + props.memoryLimitMiB = 4096; } return props; From cf0062ddc6dbec6623ba8675f7f82ed4dbcd7cd9 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Fri, 8 Nov 2024 16:57:24 -0800 Subject: [PATCH 32/41] [TM-1451] Unit test for service.getIndicators() --- .../site-polygons.service.spec.ts | 50 +++++++++++++++++++ .../src/lib/entities/tree-species.entity.ts | 16 +++--- libs/database/src/lib/factories/index.ts | 15 ++++-- ...dicator-output-field-monitoring.factory.ts | 14 ++++++ .../indicator-output-hectares.factory.ts | 14 ++++++ .../indicator-output-msu-carbon.factory.ts | 13 +++++ .../indicator-output-tree-count.factory.ts | 21 ++++++++ ...ndicator-output-tree-cover-loss.factory.ts | 13 +++++ .../indicator-output-tree-cover.factory.ts | 13 +++++ .../lib/factories/polygon-geometry.factory.ts | 22 ++++++++ .../src/lib/factories/site-polygon.factory.ts | 41 +++++++++++++++ .../src/lib/factories/site.factory.ts | 6 +++ 12 files changed, 229 insertions(+), 9 deletions(-) create mode 100644 apps/research-service/src/site-polygons/site-polygons.service.spec.ts create mode 100644 libs/database/src/lib/factories/indicator-output-field-monitoring.factory.ts create mode 100644 libs/database/src/lib/factories/indicator-output-hectares.factory.ts create mode 100644 libs/database/src/lib/factories/indicator-output-msu-carbon.factory.ts create mode 100644 libs/database/src/lib/factories/indicator-output-tree-count.factory.ts create mode 100644 libs/database/src/lib/factories/indicator-output-tree-cover-loss.factory.ts create mode 100644 libs/database/src/lib/factories/indicator-output-tree-cover.factory.ts create mode 100644 libs/database/src/lib/factories/polygon-geometry.factory.ts create mode 100644 libs/database/src/lib/factories/site-polygon.factory.ts create mode 100644 libs/database/src/lib/factories/site.factory.ts diff --git a/apps/research-service/src/site-polygons/site-polygons.service.spec.ts b/apps/research-service/src/site-polygons/site-polygons.service.spec.ts new file mode 100644 index 0000000..4e553dd --- /dev/null +++ b/apps/research-service/src/site-polygons/site-polygons.service.spec.ts @@ -0,0 +1,50 @@ +import { SitePolygonsService } from "./site-polygons.service"; +import { Test, TestingModule } from "@nestjs/testing"; +import { + IndicatorOutputFieldMonitoringFactory, + IndicatorOutputHectaresFactory, + IndicatorOutputMsuCarbonFactory, + IndicatorOutputTreeCountFactory, + IndicatorOutputTreeCoverFactory, + IndicatorOutputTreeCoverLossFactory, + SitePolygonFactory +} from "@terramatch-microservices/database/factories"; +import { Indicator } from "@terramatch-microservices/database/entities"; + +describe("SitePolygonsService", () => { + let service: SitePolygonsService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [SitePolygonsService] + }).compile(); + + service = module.get(SitePolygonsService); + }); + + afterEach(() => { + jest.restoreAllMocks(); + }); + + it("should return all indicators", async () => { + const sitePolygon = await SitePolygonFactory.create(); + await IndicatorOutputFieldMonitoringFactory.create({ sitePolygonId: sitePolygon.id }); + await IndicatorOutputHectaresFactory.create({ sitePolygonId: sitePolygon.id }); + await IndicatorOutputMsuCarbonFactory.create({ sitePolygonId: sitePolygon.id }); + await IndicatorOutputTreeCountFactory.create({ sitePolygonId: sitePolygon.id }); + await IndicatorOutputTreeCoverFactory.create({ sitePolygonId: sitePolygon.id }); + await IndicatorOutputTreeCoverLossFactory.create({ sitePolygonId: sitePolygon.id }); + + const indicators = await sitePolygon.getIndicators(); + const indicatorsDto = await service.getIndicators(sitePolygon); + expect(indicators.length).toBe(indicatorsDto.length); + + const findDto = ({ yearOfAnalysis, indicatorSlug }: Indicator) => + indicatorsDto.find(dto => dto.yearOfAnalysis === yearOfAnalysis && dto.indicatorSlug === indicatorSlug); + for (const indicator of indicators) { + const dto = findDto(indicator); + expect(dto).not.toBeNull(); + expect(indicator).toMatchObject(dto); + } + }); +}); diff --git a/libs/database/src/lib/entities/tree-species.entity.ts b/libs/database/src/lib/entities/tree-species.entity.ts index 8775fab..f62c5ed 100644 --- a/libs/database/src/lib/entities/tree-species.entity.ts +++ b/libs/database/src/lib/entities/tree-species.entity.ts @@ -13,7 +13,16 @@ import { BIGINT, BOOLEAN, STRING, UUID } from "sequelize"; import { Site } from "./site.entity"; import { SiteReport } from "./site-report.entity"; -@Table({ tableName: "v2_tree_species", underscored: true, paranoid: true }) +@Table({ + tableName: "v2_tree_species", + underscored: true, + paranoid: true, + // Multi-column @Index doesn't work with underscored column names + indexes: [ + { name: "tree_species_type_id_collection", fields: ["collection", "speciesable_id", "speciesable_type"] }, + { name: "v2_tree_species_morph_index", fields: ["speciesable_id", "speciesable_type"] } + ] +}) export class TreeSpecies extends Model { @PrimaryKey @AutoIncrement @@ -34,7 +43,6 @@ export class TreeSpecies extends Model { @AllowNull @Index("v2_tree_species_collection_index") - @Index("tree_species_type_id_collection") @Column(STRING) collection: string | null; @@ -42,13 +50,9 @@ export class TreeSpecies extends Model { hidden: boolean; @Column(STRING) - @Index("tree_species_type_id_collection") - @Index("v2_tree_species_morph_index") speciesableType: string; @Column(BIGINT.UNSIGNED) - @Index("tree_species_type_id_collection") - @Index("v2_tree_species_morph_index") speciesableId: number; @BelongsTo(() => Site, { foreignKey: "speciesableId", scope: { speciesableType: "App\\Models\\V2\\Sites\\Site" } }) diff --git a/libs/database/src/lib/factories/index.ts b/libs/database/src/lib/factories/index.ts index 0394913..a9e47ec 100644 --- a/libs/database/src/lib/factories/index.ts +++ b/libs/database/src/lib/factories/index.ts @@ -1,3 +1,12 @@ -export * from './delayed-job.factory'; -export * from './organisation.factory'; -export * from './user.factory'; +export * from "./delayed-job.factory"; +export * from "./indicator-output-field-monitoring.factory"; +export * from "./indicator-output-hectares.factory"; +export * from "./indicator-output-msu-carbon.factory"; +export * from "./indicator-output-tree-count.factory"; +export * from "./indicator-output-tree-cover.factory"; +export * from "./indicator-output-tree-cover-loss.factory"; +export * from "./organisation.factory"; +export * from "./polygon-geometry.factory"; +export * from "./site.factory"; +export * from "./site-polygon.factory"; +export * from "./user.factory"; diff --git a/libs/database/src/lib/factories/indicator-output-field-monitoring.factory.ts b/libs/database/src/lib/factories/indicator-output-field-monitoring.factory.ts new file mode 100644 index 0000000..69722fb --- /dev/null +++ b/libs/database/src/lib/factories/indicator-output-field-monitoring.factory.ts @@ -0,0 +1,14 @@ +import { IndicatorOutputFieldMonitoring } from "../entities"; +import { FactoryGirl } from "factory-girl-ts"; +import { faker } from "@faker-js/faker"; +import { SitePolygonFactory } from "./site-polygon.factory"; + +export const IndicatorOutputFieldMonitoringFactory = FactoryGirl.define(IndicatorOutputFieldMonitoring, async () => ({ + sitePolygonId: SitePolygonFactory.associate("id"), + indicatorSlug: "fieldMonitoring", + yearOfAnalysis: faker.date.past({ years: 5 }).getFullYear(), + treeCount: faker.number.int({ min: 10, max: 10000 }), + projectPhase: "Baseline", + species: "Adansonia", + survivalRate: faker.number.int({ min: 30, max: 90 }) +})); diff --git a/libs/database/src/lib/factories/indicator-output-hectares.factory.ts b/libs/database/src/lib/factories/indicator-output-hectares.factory.ts new file mode 100644 index 0000000..c4ebb47 --- /dev/null +++ b/libs/database/src/lib/factories/indicator-output-hectares.factory.ts @@ -0,0 +1,14 @@ +import { IndicatorOutputHectares } from "../entities"; +import { FactoryGirl } from "factory-girl-ts"; +import { faker } from "@faker-js/faker"; +import { SitePolygonFactory } from "./site-polygon.factory"; + +const SLUGS = ["restorationByEcoRegion", "restorationByStrategy", "restorationByLandUse"]; +const TYPES = ["Direct-Seeding", "Agroforest", "Tree-Planting"]; + +export const IndicatorOutputHectaresFactory = FactoryGirl.define(IndicatorOutputHectares, async () => ({ + sitePolygonId: SitePolygonFactory.associate("id"), + indicatorSlug: faker.helpers.arrayElement(SLUGS), + yearOfAnalysis: faker.date.past({ years: 5 }).getFullYear(), + value: { [faker.helpers.arrayElement(TYPES)]: faker.number.float({ min: 0.1, max: 0.5 }) } +})); diff --git a/libs/database/src/lib/factories/indicator-output-msu-carbon.factory.ts b/libs/database/src/lib/factories/indicator-output-msu-carbon.factory.ts new file mode 100644 index 0000000..c4424ec --- /dev/null +++ b/libs/database/src/lib/factories/indicator-output-msu-carbon.factory.ts @@ -0,0 +1,13 @@ +import { IndicatorOutputMsuCarbon } from "../entities"; +import { FactoryGirl } from "factory-girl-ts"; +import { faker } from "@faker-js/faker"; +import { SitePolygonFactory } from "./site-polygon.factory"; + +export const IndicatorOutputMsuCarbonFactory = FactoryGirl.define(IndicatorOutputMsuCarbon, async () => ({ + sitePolygonId: SitePolygonFactory.associate("id"), + indicatorSlug: "msuCarbon", + yearOfAnalysis: faker.date.past({ years: 5 }).getFullYear(), + carbonOutput: faker.number.float({ min: 0.2, max: 0.4 }), + projectPhase: "Baseline", + confidence: faker.number.float({ min: 30, max: 60 }) +})); diff --git a/libs/database/src/lib/factories/indicator-output-tree-count.factory.ts b/libs/database/src/lib/factories/indicator-output-tree-count.factory.ts new file mode 100644 index 0000000..f09457b --- /dev/null +++ b/libs/database/src/lib/factories/indicator-output-tree-count.factory.ts @@ -0,0 +1,21 @@ +import { IndicatorOutputTreeCount } from "../entities"; +import { FactoryGirl } from "factory-girl-ts"; +import { faker } from "@faker-js/faker"; +import { SitePolygonFactory } from "./site-polygon.factory"; + +const SLUGS = ["treeCount", "earlyTreeVerification"]; + +export const IndicatorOutputTreeCountFactory = FactoryGirl.define(IndicatorOutputTreeCount, async () => ({ + sitePolygonId: SitePolygonFactory.associate("id"), + indicatorSlug: faker.helpers.arrayElement(SLUGS), + yearOfAnalysis: faker.date.past({ years: 5 }).getFullYear(), + surveyType: "Remote Sensing", + surveyId: faker.number.int({ min: 100, max: 900 }), + treeCount: faker.number.int({ min: 1, max: 10000 }), + uncertaintyType: "foo", // TBD + imagerySource: "Maxar", + imageryId: faker.internet.url(), + collectionDate: faker.date.past({ years: 5 }), + projectPhase: "Midpoint", + confidence: faker.number.float({ min: 30, max: 60 }) +})); diff --git a/libs/database/src/lib/factories/indicator-output-tree-cover-loss.factory.ts b/libs/database/src/lib/factories/indicator-output-tree-cover-loss.factory.ts new file mode 100644 index 0000000..7d62c56 --- /dev/null +++ b/libs/database/src/lib/factories/indicator-output-tree-cover-loss.factory.ts @@ -0,0 +1,13 @@ +import { IndicatorOutputTreeCoverLoss } from "../entities"; +import { FactoryGirl } from "factory-girl-ts"; +import { faker } from "@faker-js/faker"; +import { SitePolygonFactory } from "./site-polygon.factory"; + +const SLUGS = ["treeCoverLoss", "treeCoverLossFires"]; + +export const IndicatorOutputTreeCoverLossFactory = FactoryGirl.define(IndicatorOutputTreeCoverLoss, async () => ({ + sitePolygonId: SitePolygonFactory.associate("id"), + indicatorSlug: faker.helpers.arrayElement(SLUGS), + yearOfAnalysis: faker.date.past({ years: 5 }).getFullYear(), + value: { [faker.date.past({ years: 5 }).getFullYear()]: faker.number.float({ min: 0.1, max: 0.3 }) } +})); diff --git a/libs/database/src/lib/factories/indicator-output-tree-cover.factory.ts b/libs/database/src/lib/factories/indicator-output-tree-cover.factory.ts new file mode 100644 index 0000000..8311ecd --- /dev/null +++ b/libs/database/src/lib/factories/indicator-output-tree-cover.factory.ts @@ -0,0 +1,13 @@ +import { IndicatorOutputTreeCover } from "../entities"; +import { FactoryGirl } from "factory-girl-ts"; +import { faker } from "@faker-js/faker"; +import { SitePolygonFactory } from "./site-polygon.factory"; + +export const IndicatorOutputTreeCoverFactory = FactoryGirl.define(IndicatorOutputTreeCover, async () => ({ + sitePolygonId: SitePolygonFactory.associate("id"), + indicatorSlug: "treeCover", + yearOfAnalysis: faker.date.past({ years: 5 }).getFullYear(), + percentCover: faker.number.int({ min: 30, max: 60 }), + projectPhase: "Baseline", + plusMinusPercent: faker.number.int({ min: 30, max: 60 }) +})); diff --git a/libs/database/src/lib/factories/polygon-geometry.factory.ts b/libs/database/src/lib/factories/polygon-geometry.factory.ts new file mode 100644 index 0000000..4228530 --- /dev/null +++ b/libs/database/src/lib/factories/polygon-geometry.factory.ts @@ -0,0 +1,22 @@ +import { FactoryGirl } from "factory-girl-ts"; +import { PolygonGeometry } from "../entities"; +import { UserFactory } from "./user.factory"; + +// The shortest polygon defined in the prod DB as of the writing of this test. +const POLYGON = { + type: "Polygon", + coordinates: [ + [ + [104.14293058113105, 13.749724096039358], + [104.68941630988292, 13.586722290863463], + [104.40664352872176, 13.993692766531538], + [104.14293058113105, 13.749724096039358] + ] + ] +}; + +export const PolygonGeometryFactory = FactoryGirl.define(PolygonGeometry, async () => ({ + uuid: crypto.randomUUID(), + polygon: POLYGON, + createdBy: UserFactory.associate("id") +})); diff --git a/libs/database/src/lib/factories/site-polygon.factory.ts b/libs/database/src/lib/factories/site-polygon.factory.ts new file mode 100644 index 0000000..56dce31 --- /dev/null +++ b/libs/database/src/lib/factories/site-polygon.factory.ts @@ -0,0 +1,41 @@ +import { SitePolygon } from "../entities"; +import { FactoryGirl } from "factory-girl-ts"; +import { SiteFactory } from "./site.factory"; +import { faker } from "@faker-js/faker"; +import { UserFactory } from "./user.factory"; +import { PolygonGeometryFactory } from "./polygon-geometry.factory"; + +const PRACTICES = [ + "agroforestry", + "planting", + "enrichment", + "applied-nucleation", + "direct-seeding", + "assisted-natural-regeneration" +]; + +const TARGET_SYS = ["riparian-area-or-wetland", "woodlot-or-plantation", "mangrove", "urban-forest", "agroforestry"]; + +const DISTR = ["single-line", "partial", "full-enrichment"]; + +export const SitePolygonFactory = FactoryGirl.define(SitePolygon, async () => { + const uuid = crypto.randomUUID(); + const name = faker.lorem.words({ min: 3, max: 7 }); + const createdBy = UserFactory.associate("id"); + return { + uuid, + primaryUuid: uuid, + siteUuid: SiteFactory.associate("uuid"), + polygonUuid: PolygonGeometryFactory.associate("uuid"), + polyName: name, + practice: faker.helpers.arrayElements(PRACTICES, { min: 1, max: 4 }).join(","), + targetSys: faker.helpers.arrayElements(TARGET_SYS, { min: 1, max: 3 }).join(","), + distr: faker.helpers.arrayElements(DISTR, { min: 1, max: 2 }).join(","), + numTrees: faker.number.int({ min: 0, max: 1000000 }), + status: "submitted", + source: "terramatch", + createdBy: createdBy.get("id"), + isActive: true, + versionName: name + }; +}); diff --git a/libs/database/src/lib/factories/site.factory.ts b/libs/database/src/lib/factories/site.factory.ts new file mode 100644 index 0000000..2e6584f --- /dev/null +++ b/libs/database/src/lib/factories/site.factory.ts @@ -0,0 +1,6 @@ +import { Site } from "../entities"; +import { FactoryGirl } from "factory-girl-ts"; + +export const SiteFactory = FactoryGirl.define(Site, async () => ({ + uuid: crypto.randomUUID() +})); From 451c05a620132f1baa312e8e9f2b9ec2a04f3dbb Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Sun, 10 Nov 2024 15:01:10 -0800 Subject: [PATCH 33/41] [TM-1451] Unit test for getEstablishmentTreeSpecies --- .../site-polygons.service.spec.ts | 20 +++++++++++++++- .../src/lib/factories/tree-species.factory.ts | 24 +++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 libs/database/src/lib/factories/tree-species.factory.ts diff --git a/apps/research-service/src/site-polygons/site-polygons.service.spec.ts b/apps/research-service/src/site-polygons/site-polygons.service.spec.ts index 4e553dd..bb5f1a7 100644 --- a/apps/research-service/src/site-polygons/site-polygons.service.spec.ts +++ b/apps/research-service/src/site-polygons/site-polygons.service.spec.ts @@ -9,7 +9,8 @@ import { IndicatorOutputTreeCoverLossFactory, SitePolygonFactory } from "@terramatch-microservices/database/factories"; -import { Indicator } from "@terramatch-microservices/database/entities"; +import { Indicator, TreeSpecies } from "@terramatch-microservices/database/entities"; +import { TreeSpeciesFactory } from "@terramatch-microservices/database/factories/tree-species.factory"; describe("SitePolygonsService", () => { let service: SitePolygonsService; @@ -47,4 +48,21 @@ describe("SitePolygonsService", () => { expect(indicator).toMatchObject(dto); } }); + + it("should return all establishment tree species", async () => { + const sitePolygon = await SitePolygonFactory.create(); + const site = await sitePolygon.loadSite(); + await TreeSpeciesFactory.forSite.createMany(3, { speciesableId: site.id }); + + const treeSpecies = await site.loadTreeSpecies(); + const treeSpeciesDto = await service.getEstablishmentTreeSpecies(sitePolygon); + expect(treeSpeciesDto.length).toBe(treeSpecies.length); + + const findDto = ({ name }: TreeSpecies) => treeSpeciesDto.find(dto => dto.name === name); + for (const tree of treeSpecies) { + const dto = findDto(tree); + expect(dto).not.toBeNull(); + expect(tree).toMatchObject(dto); + } + }); }); diff --git a/libs/database/src/lib/factories/tree-species.factory.ts b/libs/database/src/lib/factories/tree-species.factory.ts new file mode 100644 index 0000000..b842594 --- /dev/null +++ b/libs/database/src/lib/factories/tree-species.factory.ts @@ -0,0 +1,24 @@ +import { faker } from "@faker-js/faker"; +import { SiteFactory } from "./site.factory"; +import { FactoryGirl } from "factory-girl-ts"; +import { TreeSpecies } from "../entities"; + +const defaultAttributesFactory = async () => ({ + uuid: crypto.randomUUID(), + name: faker.lorem.words(2), + amount: faker.number.int({ min: 10, max: 1000 }), + collection: "tree-planted" +}); + +export const TreeSpeciesFactory = { + forSite: FactoryGirl.define(TreeSpecies, async () => ({ + ...(await defaultAttributesFactory()), + speciesableType: "App\\Models\\V2\\Sites\\Site", + speciesableId: SiteFactory.associate("id") + })) +}; + +// export const TreeSpeciesForSiteReportFactory = TreeSpeciesFactory.extendParams({ +// speciesableType: "App\\Models\\V2\\Sites\\SiteReport", +// speciesableId: SiteReportFactory.associate("id") +// }); From c74da725382f142ad84a1c3658ba16228d7f223b Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Sun, 10 Nov 2024 15:28:55 -0800 Subject: [PATCH 34/41] [TM-1451] Unit test for getReportingPeriods. --- .../site-polygons.service.spec.ts | 21 +++++++++++++++++-- libs/database/src/lib/factories/index.ts | 2 ++ .../src/lib/factories/site-report.factory.ts | 15 +++++++++++++ .../src/lib/factories/tree-species.factory.ts | 12 ++++++----- package-lock.json | 9 +++++++- package.json | 2 ++ 6 files changed, 53 insertions(+), 8 deletions(-) create mode 100644 libs/database/src/lib/factories/site-report.factory.ts diff --git a/apps/research-service/src/site-polygons/site-polygons.service.spec.ts b/apps/research-service/src/site-polygons/site-polygons.service.spec.ts index bb5f1a7..2ea88c0 100644 --- a/apps/research-service/src/site-polygons/site-polygons.service.spec.ts +++ b/apps/research-service/src/site-polygons/site-polygons.service.spec.ts @@ -7,10 +7,11 @@ import { IndicatorOutputTreeCountFactory, IndicatorOutputTreeCoverFactory, IndicatorOutputTreeCoverLossFactory, - SitePolygonFactory + SitePolygonFactory, + SiteReportFactory, + TreeSpeciesFactory } from "@terramatch-microservices/database/factories"; import { Indicator, TreeSpecies } from "@terramatch-microservices/database/entities"; -import { TreeSpeciesFactory } from "@terramatch-microservices/database/factories/tree-species.factory"; describe("SitePolygonsService", () => { let service: SitePolygonsService; @@ -65,4 +66,20 @@ describe("SitePolygonsService", () => { expect(tree).toMatchObject(dto); } }); + + it("should return all reporting periods", async () => { + const sitePolygon = await SitePolygonFactory.create(); + const site = await sitePolygon.loadSite(); + await SiteReportFactory.createMany(2, { siteId: site.id }); + const siteReports = await site.loadSiteReports(); + await TreeSpeciesFactory.forSiteReport.createMany(3, { speciesableId: siteReports[0].id }); + await TreeSpeciesFactory.forSiteReport.createMany(5, { speciesableId: siteReports[1].id }); + + await siteReports[0].loadTreeSpecies(); + await siteReports[1].loadTreeSpecies(); + const reportingPeriodsDto = await service.getReportingPeriods(sitePolygon); + expect(reportingPeriodsDto.length).toBe(siteReports.length); + expect(siteReports[0]).toMatchObject(reportingPeriodsDto[0]); + expect(siteReports[1]).toMatchObject(reportingPeriodsDto[1]); + }); }); diff --git a/libs/database/src/lib/factories/index.ts b/libs/database/src/lib/factories/index.ts index a9e47ec..198408d 100644 --- a/libs/database/src/lib/factories/index.ts +++ b/libs/database/src/lib/factories/index.ts @@ -9,4 +9,6 @@ export * from "./organisation.factory"; export * from "./polygon-geometry.factory"; export * from "./site.factory"; export * from "./site-polygon.factory"; +export * from "./site-report.factory"; +export * from "./tree-species.factory"; export * from "./user.factory"; diff --git a/libs/database/src/lib/factories/site-report.factory.ts b/libs/database/src/lib/factories/site-report.factory.ts new file mode 100644 index 0000000..4ba73f1 --- /dev/null +++ b/libs/database/src/lib/factories/site-report.factory.ts @@ -0,0 +1,15 @@ +import { SiteReport } from "../entities"; +import { FactoryGirl } from "factory-girl-ts"; +import { SiteFactory } from "./site.factory"; +import { faker } from "@faker-js/faker"; +import { DateTime } from "luxon"; + +export const SiteReportFactory = FactoryGirl.define(SiteReport, async () => { + const dueAt = faker.date.past({ years: 2 }); + return { + uuid: crypto.randomUUID(), + siteId: SiteFactory.associate("id"), + dueAt, + submittedAt: faker.date.between({ from: dueAt, to: DateTime.fromJSDate(dueAt).plus({ days: 14 }).toJSDate() }) + }; +}); diff --git a/libs/database/src/lib/factories/tree-species.factory.ts b/libs/database/src/lib/factories/tree-species.factory.ts index b842594..1a7304e 100644 --- a/libs/database/src/lib/factories/tree-species.factory.ts +++ b/libs/database/src/lib/factories/tree-species.factory.ts @@ -2,6 +2,7 @@ import { faker } from "@faker-js/faker"; import { SiteFactory } from "./site.factory"; import { FactoryGirl } from "factory-girl-ts"; import { TreeSpecies } from "../entities"; +import { SiteReportFactory } from "./site-report.factory"; const defaultAttributesFactory = async () => ({ uuid: crypto.randomUUID(), @@ -15,10 +16,11 @@ export const TreeSpeciesFactory = { ...(await defaultAttributesFactory()), speciesableType: "App\\Models\\V2\\Sites\\Site", speciesableId: SiteFactory.associate("id") + })), + + forSiteReport: FactoryGirl.define(TreeSpecies, async () => ({ + ...(await defaultAttributesFactory()), + speciesableType: "App\\Models\\V2\\Sites\\SiteReport", + speciesableId: SiteReportFactory.associate("id") })) }; - -// export const TreeSpeciesForSiteReportFactory = TreeSpeciesFactory.extendParams({ -// speciesableType: "App\\Models\\V2\\Sites\\SiteReport", -// speciesableId: SiteReportFactory.associate("id") -// }); diff --git a/package-lock.json b/package-lock.json index 356e694..fbab6f8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -25,6 +25,7 @@ "class-validator": "^0.14.1", "geojson": "^0.5.0", "lodash": "^4.17.21", + "luxon": "^3.5.0", "mariadb": "^3.3.2", "mysql2": "^3.11.2", "nestjs-request-context": "^3.0.0", @@ -55,6 +56,7 @@ "@types/geojson": "^7946.0.14", "@types/jest": "^29.5.12", "@types/lodash": "^4.17.13", + "@types/luxon": "^3.4.2", "@types/node": "~18.16.9", "@types/supertest": "^6.0.2", "@types/validator": "^13.12.2", @@ -4695,6 +4697,12 @@ "integrity": "sha512-lfx+dftrEZcdBPczf9d0Qv0x+j/rfNCMuC6OcfXmO8gkfeNAY88PgKUbvG56whcN23gc27yenwF6oJZXGFpYxg==", "dev": true }, + "node_modules/@types/luxon": { + "version": "3.4.2", + "resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.4.2.tgz", + "integrity": "sha512-TifLZlFudklWlMBfhubvgqTXRzLDI5pCbGa4P8a3wPyUQSW+1xQ5eDsreP9DWHX3tjq1ke96uYG/nwundroWcA==", + "dev": true + }, "node_modules/@types/methods": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/@types/methods/-/methods-1.1.4.tgz", @@ -11153,7 +11161,6 @@ "version": "3.5.0", "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.5.0.tgz", "integrity": "sha512-rh+Zjr6DNfUYR3bPwJEnuwDdqMbxZW7LOQfUN4B54+Cl+0o5zaU9RJ6bcidfDtC1cWCZXQ+nvX8bf6bAji37QQ==", - "dev": true, "engines": { "node": ">=12" } diff --git a/package.json b/package.json index a87214c..4e169e9 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ "class-validator": "^0.14.1", "geojson": "^0.5.0", "lodash": "^4.17.21", + "luxon": "^3.5.0", "mariadb": "^3.3.2", "mysql2": "^3.11.2", "nestjs-request-context": "^3.0.0", @@ -55,6 +56,7 @@ "@types/geojson": "^7946.0.14", "@types/jest": "^29.5.12", "@types/lodash": "^4.17.13", + "@types/luxon": "^3.4.2", "@types/node": "~18.16.9", "@types/supertest": "^6.0.2", "@types/validator": "^13.12.2", From 69ac4fab21d75bef7ad8718093b3bae5160d5e64 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Sun, 10 Nov 2024 15:48:16 -0800 Subject: [PATCH 35/41] [TM-1451] Complete coverage for the service. --- .../site-polygons.service.spec.ts | 41 ++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/apps/research-service/src/site-polygons/site-polygons.service.spec.ts b/apps/research-service/src/site-polygons/site-polygons.service.spec.ts index 2ea88c0..d5eb998 100644 --- a/apps/research-service/src/site-polygons/site-polygons.service.spec.ts +++ b/apps/research-service/src/site-polygons/site-polygons.service.spec.ts @@ -11,7 +11,8 @@ import { SiteReportFactory, TreeSpeciesFactory } from "@terramatch-microservices/database/factories"; -import { Indicator, TreeSpecies } from "@terramatch-microservices/database/entities"; +import { Indicator, PolygonGeometry, SitePolygon, TreeSpecies } from "@terramatch-microservices/database/entities"; +import { BadRequestException } from "@nestjs/common"; describe("SitePolygonsService", () => { let service: SitePolygonsService; @@ -82,4 +83,42 @@ describe("SitePolygonsService", () => { expect(siteReports[0]).toMatchObject(reportingPeriodsDto[0]); expect(siteReports[1]).toMatchObject(reportingPeriodsDto[1]); }); + + it("should return all polygons when there are fewer than the page size", async () => { + await SitePolygon.truncate(); + await PolygonGeometry.truncate(); + await SitePolygonFactory.createMany(15); + const query = await service.buildQuery(20); + const result = await query.execute(); + expect(result.length).toBe(15); + }); + + it("should return page size when there are more than the page size", async () => { + await SitePolygon.truncate(); + await PolygonGeometry.truncate(); + await SitePolygonFactory.createMany(15); + const query = await service.buildQuery(10); + const result = await query.execute(); + expect(result.length).toBe(10); + }); + + it("Should return only the entries after the given entry when pageAfter is provided", async () => { + await SitePolygon.truncate(); + await PolygonGeometry.truncate(); + await SitePolygonFactory.createMany(15); + const first = await SitePolygon.findOne(); + const query = await service.buildQuery(20, first.uuid); + const result = await query.execute(); + expect(result.length).toBe(14); + }); + + it("Should throw when pageAfter polygon not found", () => { + expect(service.buildQuery(20, "asdfasdf")).rejects.toThrow(BadRequestException); + }); + + it("Should return empty arrays from utility methods if no associated records exist", async () => { + const sitePolygon = await SitePolygonFactory.create({ siteUuid: null }); + expect(await service.getEstablishmentTreeSpecies(sitePolygon)).toEqual([]); + expect(await service.getReportingPeriods(sitePolygon)).toEqual([]); + }); }); From 0a049076c96c6dda952a6a809b8a6613630cf7b6 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Sun, 10 Nov 2024 16:10:51 -0800 Subject: [PATCH 36/41] [TM-1451] Controller error cases. --- .../site-polygons.controller.spec.ts | 64 +++++++++++++++++++ .../site-polygons/site-polygons.controller.ts | 2 +- .../site-polygons/site-polygons.service.ts | 2 +- .../src/users/users.controller.spec.ts | 60 ++++++++--------- 4 files changed, 94 insertions(+), 34 deletions(-) create mode 100644 apps/research-service/src/site-polygons/site-polygons.controller.spec.ts diff --git a/apps/research-service/src/site-polygons/site-polygons.controller.spec.ts b/apps/research-service/src/site-polygons/site-polygons.controller.spec.ts new file mode 100644 index 0000000..ef68c59 --- /dev/null +++ b/apps/research-service/src/site-polygons/site-polygons.controller.spec.ts @@ -0,0 +1,64 @@ +import { SitePolygonsController } from "./site-polygons.controller"; +import { SitePolygonQueryBuilder, SitePolygonsService } from "./site-polygons.service"; +import { createMock, DeepMocked } from "@golevelup/ts-jest"; +import { Test, TestingModule } from "@nestjs/testing"; +import { PolicyService } from "@terramatch-microservices/common"; +import { BadRequestException, NotImplementedException, UnauthorizedException } from "@nestjs/common"; + +describe("SitePolygonsController", () => { + let controller: SitePolygonsController; + let sitePolygonService: DeepMocked; + let policyService: DeepMocked; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + controllers: [SitePolygonsController], + providers: [ + { provide: SitePolygonsService, useValue: (sitePolygonService = createMock()) }, + { provide: PolicyService, useValue: (policyService = createMock()) } + ] + }).compile(); + + controller = module.get(SitePolygonsController); + }); + + afterEach(() => { + jest.restoreAllMocks(); + }); + + describe("findMany", () => { + it("should should throw an error if the policy does not authorize", async () => { + policyService.authorize.mockRejectedValue(new UnauthorizedException()); + await expect(controller.findMany({})).rejects.toThrow(UnauthorizedException); + }); + + it("should throw an error if the page size is invalid", async () => { + policyService.authorize.mockResolvedValue(undefined); + await expect(controller.findMany({ page: { size: 300 } })).rejects.toThrow(BadRequestException); + await expect(controller.findMany({ page: { size: -1 } })).rejects.toThrow(BadRequestException); + }); + + it("should throw an error if the page after is invalid", async () => { + policyService.authorize.mockResolvedValue(undefined); + sitePolygonService.buildQuery.mockRejectedValue(new BadRequestException()); + await expect(controller.findMany({ page: { after: "asdfasdf" } })).rejects.toThrow(BadRequestException); + }); + + it("Returns a valid value if the request is valid", async () => { + policyService.authorize.mockResolvedValue(undefined); + const Builder = { execute: jest.fn() }; + Builder.execute.mockResolvedValue([]); + sitePolygonService.buildQuery.mockResolvedValue(Builder as unknown as SitePolygonQueryBuilder); + const result = await controller.findMany({}); + expect(result.meta).not.toBe(null); + expect(result.meta.page.total).toBe(0); + expect(result.meta.page.cursor).toBeUndefined(); + }); + }); + + describe("bulkUpdate", () => { + it("Should throw", async () => { + await expect(controller.bulkUpdate(null)).rejects.toThrow(NotImplementedException); + }); + }); +}); diff --git a/apps/research-service/src/site-polygons/site-polygons.controller.ts b/apps/research-service/src/site-polygons/site-polygons.controller.ts index aac96c8..c24b5c5 100644 --- a/apps/research-service/src/site-polygons/site-polygons.controller.ts +++ b/apps/research-service/src/site-polygons/site-polygons.controller.ts @@ -49,7 +49,7 @@ export class SitePolygonsController { @JsonApiResponse({ data: { type: SitePolygonDto }, pagination: true }) @ApiException(() => UnauthorizedException, { description: "Authentication failed." }) @ApiException(() => BadRequestException, { description: "Pagination values are invalid." }) - async findMany(@Query() query?: SitePolygonQueryDto): Promise { + async findMany(@Query() query: SitePolygonQueryDto): Promise { await this.policyService.authorize("readAll", SitePolygon); const { size: pageSize = MAX_PAGE_SIZE, after: pageAfter } = query.page ?? {}; diff --git a/apps/research-service/src/site-polygons/site-polygons.service.ts b/apps/research-service/src/site-polygons/site-polygons.service.ts index bbf6249..962b992 100644 --- a/apps/research-service/src/site-polygons/site-polygons.service.ts +++ b/apps/research-service/src/site-polygons/site-polygons.service.ts @@ -6,7 +6,7 @@ import { INDICATOR_DTOS } from "./dto/indicators.dto"; import { ModelPropertiesAccessor } from "@nestjs/swagger/dist/services/model-properties-accessor"; import { pick } from "lodash"; -class SitePolygonQueryBuilder { +export class SitePolygonQueryBuilder { private findOptions: FindOptions> = { include: [ "indicatorsFieldMonitoring", diff --git a/apps/user-service/src/users/users.controller.spec.ts b/apps/user-service/src/users/users.controller.spec.ts index f4ae475..d19a9a6 100644 --- a/apps/user-service/src/users/users.controller.spec.ts +++ b/apps/user-service/src/users/users.controller.spec.ts @@ -1,21 +1,19 @@ -import { Test, TestingModule } from '@nestjs/testing'; -import { UsersController } from './users.controller'; -import { PolicyService } from '@terramatch-microservices/common'; -import { createMock, DeepMocked } from '@golevelup/ts-jest'; -import { NotFoundException, UnauthorizedException } from '@nestjs/common'; -import { OrganisationFactory, UserFactory } from '@terramatch-microservices/database/factories'; -import { Relationship, Resource } from '@terramatch-microservices/common/util'; +import { Test, TestingModule } from "@nestjs/testing"; +import { UsersController } from "./users.controller"; +import { PolicyService } from "@terramatch-microservices/common"; +import { createMock, DeepMocked } from "@golevelup/ts-jest"; +import { NotFoundException, UnauthorizedException } from "@nestjs/common"; +import { OrganisationFactory, UserFactory } from "@terramatch-microservices/database/factories"; +import { Relationship, Resource } from "@terramatch-microservices/common/util"; -describe('UsersController', () => { +describe("UsersController", () => { let controller: UsersController; let policyService: DeepMocked; beforeEach(async () => { const module: TestingModule = await Test.createTestingModule({ controllers: [UsersController], - providers: [ - { provide: PolicyService, useValue: policyService = createMock() }, - ] + providers: [{ provide: PolicyService, useValue: (policyService = createMock()) }] }).compile(); controller = module.get(UsersController); @@ -23,62 +21,60 @@ describe('UsersController', () => { afterEach(() => { jest.restoreAllMocks(); - }) + }); - it('should throw not found if the user is not found', async () => { - await expect(controller.findOne('0', { authenticatedUserId: 1 })).rejects - .toThrow(NotFoundException); + it("should throw not found if the user is not found", async () => { + await expect(controller.findOne("0", { authenticatedUserId: 1 })).rejects.toThrow(NotFoundException); }); - it('should throw an error if the policy does not authorize', async () => { - policyService.authorize.mockRejectedValue(new UnauthorizedException()) + it("should throw an error if the policy does not authorize", async () => { + policyService.authorize.mockRejectedValue(new UnauthorizedException()); const { id } = await UserFactory.create(); - await expect(controller.findOne(`${id}`, { authenticatedUserId: 1 })).rejects - .toThrow(UnauthorizedException); + await expect(controller.findOne(`${id}`, { authenticatedUserId: 1 })).rejects.toThrow(UnauthorizedException); }); it('should return the currently logged in user if the id is "me"', async () => { const { id, uuid } = await UserFactory.create(); - const result = await controller.findOne('me', { authenticatedUserId: id }); + const result = await controller.findOne("me", { authenticatedUserId: id }); expect((result.data as Resource).id).toBe(uuid); }); - it('should return the indicated user if the logged in user is allowed to access', async () => { + it("should return the indicated user if the logged in user is allowed to access", async () => { policyService.authorize.mockResolvedValue(undefined); const { id, uuid } = await UserFactory.create(); const result = await controller.findOne(`${id}`, { authenticatedUserId: id + 1 }); expect((result.data as Resource).id).toBe(uuid); }); - it('should return a document without includes if there is no org', async () => { + it("should return a document without includes if there is no org", async () => { const { id } = await UserFactory.create(); - const result = await controller.findOne('me', { authenticatedUserId: id }); + const result = await controller.findOne("me", { authenticatedUserId: id }); expect(result.included).not.toBeDefined(); }); - it('should include the primary org for the user', async () => { + it("should include the primary org for the user", async () => { const user = await UserFactory.create(); const org = await OrganisationFactory.create(); - await user.$add('organisationsConfirmed', org); - const result = await controller.findOne('me', { authenticatedUserId: user.id }); + await user.$add("organisationsConfirmed", org); + const result = await controller.findOne("me", { authenticatedUserId: user.id }); expect(result.included).toHaveLength(1); - expect(result.included[0]).toMatchObject({ type: 'organisations', id: org.uuid }); + expect(result.included[0]).toMatchObject({ type: "organisations", id: org.uuid }); const data = result.data as Resource; expect(data.relationships.org).toBeDefined(); const relationship = data.relationships.org.data as Relationship; - expect(relationship).toMatchObject({ type: 'organisations', id: org.uuid, meta: { userStatus: 'approved' } }); + expect(relationship).toMatchObject({ type: "organisations", id: org.uuid, meta: { userStatus: "approved" } }); }); it('should return "na" for userStatus if there is no many to many relationship', async () => { const user = await UserFactory.create(); const org = await OrganisationFactory.create(); - await user.$set('organisation', org); - const result = await controller.findOne('me', { authenticatedUserId: user.id }); + await user.$set("organisation", org); + const result = await controller.findOne("me", { authenticatedUserId: user.id }); expect(result.included).toHaveLength(1); - expect(result.included[0]).toMatchObject({ type: 'organisations', id: org.uuid }); + expect(result.included[0]).toMatchObject({ type: "organisations", id: org.uuid }); const data = result.data as Resource; expect(data.relationships.org).toBeDefined(); const relationship = data.relationships.org.data as Relationship; - expect(relationship).toMatchObject({ type: 'organisations', id: org.uuid, meta: { userStatus: 'na' } }); + expect(relationship).toMatchObject({ type: "organisations", id: org.uuid, meta: { userStatus: "na" } }); }); }); From 0a2867cf5197738f7054e80afaf1227906a486f4 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Sun, 10 Nov 2024 20:48:36 -0800 Subject: [PATCH 37/41] [TM-1451] Full coverage for the research service. --- apps/research-service/jest.config.ts | 13 +++++++------ .../site-polygons.controller.spec.ts | 18 +++++++++++++----- libs/common/src/lib/util/json-api-builder.ts | 3 ++- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/apps/research-service/jest.config.ts b/apps/research-service/jest.config.ts index a670078..78bb029 100644 --- a/apps/research-service/jest.config.ts +++ b/apps/research-service/jest.config.ts @@ -1,11 +1,12 @@ /* eslint-disable */ export default { - displayName: 'research-service', - preset: '../../jest.preset.js', - testEnvironment: 'node', + displayName: "research-service", + preset: "../../jest.preset.js", + testEnvironment: "node", transform: { - '^.+\\.[tj]s$': ['ts-jest', { tsconfig: '/tsconfig.spec.json' }], + "^.+\\.[tj]s$": ["ts-jest", { tsconfig: "/tsconfig.spec.json" }] }, - moduleFileExtensions: ['ts', 'js', 'html'], - coverageDirectory: '../../coverage/apps/research-service', + moduleFileExtensions: ["ts", "js", "html"], + coveragePathIgnorePatterns: [".dto.ts"], + coverageDirectory: "../../coverage/apps/research-service" }; diff --git a/apps/research-service/src/site-polygons/site-polygons.controller.spec.ts b/apps/research-service/src/site-polygons/site-polygons.controller.spec.ts index ef68c59..9edb844 100644 --- a/apps/research-service/src/site-polygons/site-polygons.controller.spec.ts +++ b/apps/research-service/src/site-polygons/site-polygons.controller.spec.ts @@ -1,9 +1,11 @@ import { SitePolygonsController } from "./site-polygons.controller"; -import { SitePolygonQueryBuilder, SitePolygonsService } from "./site-polygons.service"; +import { SitePolygonsService } from "./site-polygons.service"; import { createMock, DeepMocked } from "@golevelup/ts-jest"; import { Test, TestingModule } from "@nestjs/testing"; import { PolicyService } from "@terramatch-microservices/common"; import { BadRequestException, NotImplementedException, UnauthorizedException } from "@nestjs/common"; +import { SitePolygonFactory } from "@terramatch-microservices/database/factories"; +import { Resource } from "@terramatch-microservices/common/util"; describe("SitePolygonsController", () => { let controller: SitePolygonsController; @@ -46,13 +48,19 @@ describe("SitePolygonsController", () => { it("Returns a valid value if the request is valid", async () => { policyService.authorize.mockResolvedValue(undefined); + const sitePolygon = await SitePolygonFactory.create(); const Builder = { execute: jest.fn() }; - Builder.execute.mockResolvedValue([]); - sitePolygonService.buildQuery.mockResolvedValue(Builder as unknown as SitePolygonQueryBuilder); + Builder.execute.mockResolvedValue([sitePolygon]); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + sitePolygonService.buildQuery.mockResolvedValue(Builder as any); const result = await controller.findMany({}); expect(result.meta).not.toBe(null); - expect(result.meta.page.total).toBe(0); - expect(result.meta.page.cursor).toBeUndefined(); + expect(result.meta.page.total).toBe(1); + expect(result.meta.page.cursor).toBe(sitePolygon.uuid); + + const resources = result.data as Resource[]; + expect(resources.length).toBe(1); + expect(resources[0].id).toBe(sitePolygon.uuid); }); }); diff --git a/libs/common/src/lib/util/json-api-builder.ts b/libs/common/src/lib/util/json-api-builder.ts index aa20eac..8f1fb2a 100644 --- a/libs/common/src/lib/util/json-api-builder.ts +++ b/libs/common/src/lib/util/json-api-builder.ts @@ -146,9 +146,10 @@ class DocumentBuilder { } serialize(): JsonApiDocument { + const singular = this.data.length === 1 && this.options?.pagination !== true; const doc: JsonApiDocument = { // Data can either be a single object or an array - data: this.data.length === 1 ? this.data[0].serialize() : this.data.map(resource => resource.serialize()) + data: singular ? this.data[0].serialize() : this.data.map(resource => resource.serialize()) }; if (this.included.length > 0) { From aa9a63edc283c293c4afeb01c46991695f0f5701 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Sun, 10 Nov 2024 21:15:26 -0800 Subject: [PATCH 38/41] [TM-1451] Coverage for the common lib. --- libs/common/src/lib/guards/auth.guard.spec.ts | 83 +++++++++++-------- .../lib/policies/site-polygon.policy.spec.ts | 33 ++++++++ 2 files changed, 81 insertions(+), 35 deletions(-) create mode 100644 libs/common/src/lib/policies/site-polygon.policy.spec.ts diff --git a/libs/common/src/lib/guards/auth.guard.spec.ts b/libs/common/src/lib/guards/auth.guard.spec.ts index 54d3c50..944bcf7 100644 --- a/libs/common/src/lib/guards/auth.guard.spec.ts +++ b/libs/common/src/lib/guards/auth.guard.spec.ts @@ -1,37 +1,40 @@ -import { AuthGuard, NoBearerAuth } from './auth.guard'; -import { Test } from '@nestjs/testing'; -import { APP_GUARD } from '@nestjs/core'; -import { createMock, DeepMocked } from '@golevelup/ts-jest'; -import { JwtService } from '@nestjs/jwt'; -import { Controller, Get, HttpStatus, INestApplication } from '@nestjs/common'; -import * as request from 'supertest'; +import { AuthGuard, NoBearerAuth } from "./auth.guard"; +import { Test } from "@nestjs/testing"; +import { APP_GUARD } from "@nestjs/core"; +import { createMock, DeepMocked } from "@golevelup/ts-jest"; +import { JwtService } from "@nestjs/jwt"; +import { Controller, Get, HttpStatus, INestApplication } from "@nestjs/common"; +import * as request from "supertest"; +import { UserFactory } from "@terramatch-microservices/database/factories"; -@Controller('test') +@Controller("test") class TestController { @Get() test() { - return 'test'; + return "test"; } @NoBearerAuth - @Get('/no-auth') + @Get("/no-auth") noAuth() { - return 'no-auth'; + return "no-auth"; } } -describe('AuthGuard', () => { +describe("AuthGuard", () => { let jwtService: DeepMocked; let app: INestApplication; beforeEach(async () => { - app = (await Test.createTestingModule({ - controllers: [TestController], - providers: [ - { provide: JwtService, useValue: jwtService = createMock() }, - { provide: APP_GUARD, useClass: AuthGuard }, - ], - }).compile()).createNestApplication(); + app = ( + await Test.createTestingModule({ + controllers: [TestController], + providers: [ + { provide: JwtService, useValue: (jwtService = createMock()) }, + { provide: APP_GUARD, useClass: AuthGuard } + ] + }).compile() + ).createNestApplication(); await app.init(); }); @@ -40,30 +43,40 @@ describe('AuthGuard', () => { jest.restoreAllMocks(); }); - it('should return an error when no auth header is present', async () => { - await request(app.getHttpServer()) - .get('/test') - .expect(HttpStatus.UNAUTHORIZED); + it("should return an error when no auth header is present", async () => { + await request(app.getHttpServer()).get("/test").expect(HttpStatus.UNAUTHORIZED); }); - it('should not return an error when a valid auth header is present', async () => { - const token = 'fake jwt token'; - jwtService.verifyAsync.mockResolvedValue({ sub: 'fakeuserid' }); + it("should not return an error when a valid auth header is present", async () => { + const token = "fake jwt token"; + jwtService.verifyAsync.mockResolvedValue({ sub: "fakeuserid" }); - await request(app.getHttpServer()) - .get('/test') - .set('Authorization', `Bearer ${token}`) - .expect(HttpStatus.OK); + await request(app.getHttpServer()).get("/test").set("Authorization", `Bearer ${token}`).expect(HttpStatus.OK); }); - it('should ignore bearer token on an endpoint with @NoBearerAuth', async () => { + it("should ignore bearer token on an endpoint with @NoBearerAuth", async () => { + await request(app.getHttpServer()).get("/test/no-auth").expect(HttpStatus.OK); + await request(app.getHttpServer()) - .get('/test/no-auth') + .get("/test/no-auth") + .set("Authorization", "Bearer fake jwt token") .expect(HttpStatus.OK); + }); + + it("should use an api key for login", async () => { + const apiKey = "fake-api-key"; + await UserFactory.create({ apiKey }); + jwtService.decode.mockReturnValue(null); + + await request(app.getHttpServer()).get("/test").set("Authorization", `Bearer ${apiKey}`).expect(HttpStatus.OK); + }); + + it("should throw when the api key is not recognized", async () => { + jwtService.decode.mockReturnValue(null); await request(app.getHttpServer()) - .get('/test/no-auth') - .set('Authorization', 'Bearer fake jwt token') - .expect(HttpStatus.OK); + .get("/test") + .set("Authorization", "Bearer foobar") + .expect(HttpStatus.UNAUTHORIZED); }); }); diff --git a/libs/common/src/lib/policies/site-polygon.policy.spec.ts b/libs/common/src/lib/policies/site-polygon.policy.spec.ts new file mode 100644 index 0000000..985d065 --- /dev/null +++ b/libs/common/src/lib/policies/site-polygon.policy.spec.ts @@ -0,0 +1,33 @@ +import { PolicyService } from "./policy.service"; +import { Test, TestingModule } from "@nestjs/testing"; +import { mockPermissions, mockUserId } from "./policy.service.spec"; +import { SitePolygon } from "@terramatch-microservices/database/entities"; +import { UnauthorizedException } from "@nestjs/common"; + +describe("SitePolygonPolicy", () => { + let service: PolicyService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [PolicyService] + }).compile(); + + service = module.get(PolicyService); + }); + + afterEach(async () => { + jest.restoreAllMocks(); + }); + + it("allows reading any polygon with polygons-manage", async () => { + mockUserId(123); + mockPermissions("polygons-manage"); + await expect(service.authorize("readAll", SitePolygon)).resolves.toBeUndefined(); + }); + + it("disallows reading polygons without polygons-manage", async () => { + mockUserId(123); + mockPermissions(); + await expect(service.authorize("readAll", SitePolygon)).rejects.toThrow(UnauthorizedException); + }); +}); From 35d9220f8216ace3504274e2b6329db766fd2a28 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Mon, 11 Nov 2024 10:38:12 -0800 Subject: [PATCH 39/41] [TM-1451] Use run-many for lint / build --- .github/workflows/pull-request.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 8b3e40e..b6bc319 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -31,9 +31,7 @@ jobs: - run: npm ci --legacy-peer-deps - - uses: nrwl/nx-set-shas@v4 - - - run: NX_CLOUD_DISTRIBUTED_EXECUTION=false npx nx affected -t lint build + - run: NX_CLOUD_DISTRIBUTED_EXECUTION=false npx nx run-many -t lint build - uses: KengoTODA/actions-setup-docker-compose@v1 with: From 391cf2eeeedc6c038d1a5b15e378f9d8d90b14c5 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Mon, 11 Nov 2024 16:35:54 -0800 Subject: [PATCH 40/41] [TM-1448] Try triggering job service build from new action --- .github/workflows/deploy-service.yml | 8 +++++ .github/workflows/deploy-services.yml | 43 +++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 .github/workflows/deploy-services.yml diff --git a/.github/workflows/deploy-service.yml b/.github/workflows/deploy-service.yml index 98ea53d..2013fe3 100644 --- a/.github/workflows/deploy-service.yml +++ b/.github/workflows/deploy-service.yml @@ -2,6 +2,14 @@ name: Service Deploy run-name: 'Service Deploy [service: ${{ inputs.service }}, env: ${{ inputs.env }}]' on: + workflow_call: + inputs: + service: + required: true + type: string + env: + required: true + type: string workflow_dispatch: inputs: service: diff --git a/.github/workflows/deploy-services.yml b/.github/workflows/deploy-services.yml new file mode 100644 index 0000000..5844f4d --- /dev/null +++ b/.github/workflows/deploy-services.yml @@ -0,0 +1,43 @@ +name: Services Deploy (all) +run-name: 'Services Deploy (all) [env: ${{ inputs.env }}]' + +on: + workflow_dispatch: + inputs: + env: + description: 'Deployment target environment' + type: choice + required: true + options: + - dev + - test + - staging + - prod + +permissions: + id-token: write + contents: read + +jobs: + job-service: + uses: ./.github/workflows/deploy-service.yml + with: + env: ${{ inputs.env }} + service: job-service + secrets: inherit + + user-service: + uses: ./.github/workflows/deploy-service.yml + with: + env: ${{ inputs.env }} + service: user-service + secrets: inherit + + research-service: + uses: ./.github/workflows/deploy-service.yml + with: + env: ${{ inputs.env }} + service: research-service + secrets: inherit + + From d7a8f1a4ff6a53a7964311b5b52223484de5deaf Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Mon, 11 Nov 2024 17:03:52 -0800 Subject: [PATCH 41/41] [TM-1448] Update the README. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index cae5048..2e22601 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,7 @@ and main branches. * For deployment to AWS: * Add a Dockerfile in the new app directory. A simple copy and modify from user-service is sufficient * Add the new service name to the "service" workflow input options in `deploy-service.yml` + * Add a new job to `deploy-services.yml` to include the new services in the "all" service deployment workflow. * In AWS: * Add ECR repositories for each env (follow the naming scheme from user-service, e.g. `terramatch-microservices/foo-service-staging`, etc) * Set the repo to Immutable