From b5b0fe2f028b041160fec005481dafff2f2591c7 Mon Sep 17 00:00:00 2001
From: Nathan Curtis <nathan.curtis@gmail.com>
Date: Fri, 15 Nov 2024 13:04:31 -0800
Subject: [PATCH 01/16] [TM-1452] Add documentation for includeTestProjects

---
 .../src/site-polygons/dto/site-polygon-query.dto.ts        | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/apps/research-service/src/site-polygons/dto/site-polygon-query.dto.ts b/apps/research-service/src/site-polygons/dto/site-polygon-query.dto.ts
index 62da244..47ccf13 100644
--- a/apps/research-service/src/site-polygons/dto/site-polygon-query.dto.ts
+++ b/apps/research-service/src/site-polygons/dto/site-polygon-query.dto.ts
@@ -78,6 +78,13 @@ export class SitePolygonQueryDto {
   @IsOptional()
   boundaryPolygon?: string;
 
+  @ApiProperty({
+    required: false,
+    default: false,
+    description: "Include polygons for test projects in the results"
+  })
+  includeTestProjects?: boolean;
+
   @ApiProperty({ name: "page", required: false, description: "Pagination information" })
   @ValidateNested()
   @IsOptional()

From b0b781253096681796e6090223115e5cfa2e8e3c Mon Sep 17 00:00:00 2001
From: Nathan Curtis <nathan.curtis@gmail.com>
Date: Mon, 18 Nov 2024 12:33:59 -0800
Subject: [PATCH 02/16] [TM-1452] Implement includeTestProjects filter.

---
 .../site-polygons/site-polygons.controller.ts |  3 +
 .../site-polygons/site-polygons.service.ts    | 34 +++++--
 .../src/lib/entities/organisation.entity.ts   | 88 ++++++++-----------
 .../src/lib/entities/project.entity.ts        |  8 +-
 libs/database/src/lib/entities/site.entity.ts |  7 +-
 5 files changed, 76 insertions(+), 64 deletions(-)

diff --git a/apps/research-service/src/site-polygons/site-polygons.controller.ts b/apps/research-service/src/site-polygons/site-polygons.controller.ts
index c24b5c5..14211b1 100644
--- a/apps/research-service/src/site-polygons/site-polygons.controller.ts
+++ b/apps/research-service/src/site-polygons/site-polygons.controller.ts
@@ -58,6 +58,9 @@ export class SitePolygonsController {
     }
 
     const queryBuilder = await this.sitePolygonService.buildQuery(pageSize, pageAfter);
+    if (query.includeTestProjects !== true) {
+      await queryBuilder.excludeTestProjects();
+    }
 
     const document = buildJsonApi({ pagination: true });
     for (const sitePolygon of await queryBuilder.execute()) {
diff --git a/apps/research-service/src/site-polygons/site-polygons.service.ts b/apps/research-service/src/site-polygons/site-polygons.service.ts
index 962b992..3637722 100644
--- a/apps/research-service/src/site-polygons/site-polygons.service.ts
+++ b/apps/research-service/src/site-polygons/site-polygons.service.ts
@@ -1,12 +1,17 @@
 import { BadRequestException, Injectable, Type } from "@nestjs/common";
-import { Site, SitePolygon, SiteReport } from "@terramatch-microservices/database/entities";
-import { Attributes, FindOptions, Op, WhereOptions } from "sequelize";
+import { Project, Site, SitePolygon, SiteReport } from "@terramatch-microservices/database/entities";
+import { Attributes, FindOptions, IncludeOptions, Op, WhereOptions } from "sequelize";
 import { IndicatorDto, ReportingPeriodDto, TreeSpeciesDto } from "./dto/site-polygon.dto";
 import { INDICATOR_DTOS } from "./dto/indicators.dto";
 import { ModelPropertiesAccessor } from "@nestjs/swagger/dist/services/model-properties-accessor";
 import { pick } from "lodash";
 
 export class SitePolygonQueryBuilder {
+  private siteJoin: IncludeOptions = {
+    model: Site,
+    include: ["treeSpecies", { model: SiteReport, include: ["treeSpecies"] }],
+    required: true
+  };
   private findOptions: FindOptions<Attributes<SitePolygon>> = {
     include: [
       "indicatorsFieldMonitoring",
@@ -16,10 +21,7 @@ export class SitePolygonQueryBuilder {
       "indicatorsTreeCover",
       "indicatorsTreeCoverLoss",
       "polygon",
-      {
-        model: Site,
-        include: ["treeSpecies", { model: SiteReport, include: ["treeSpecies"] }]
-      }
+      this.siteJoin
     ]
   };
 
@@ -27,6 +29,13 @@ export class SitePolygonQueryBuilder {
     this.findOptions.limit = pageSize;
   }
 
+  async excludeTestProjects() {
+    // avoid joining against the entire project table by doing a quick query first. The number of test projects is small
+    const testProjects = await Project.findAll({ where: { isTest: true }, attributes: ["id"] });
+    this.where({ projectId: { [Op.notIn]: testProjects.map(({ id }) => id) } }, this.siteJoin);
+    return this;
+  }
+
   async pageAfter(pageAfter: string) {
     const sitePolygon = await SitePolygon.findOne({ where: { uuid: pageAfter }, attributes: ["id"] });
     if (sitePolygon == null) throw new BadRequestException("pageAfter polygon not found");
@@ -38,9 +47,16 @@ export class SitePolygonQueryBuilder {
     return await SitePolygon.findAll(this.findOptions);
   }
 
-  private where(options: WhereOptions) {
-    if (this.findOptions.where == null) this.findOptions.where = {};
-    Object.assign(this.findOptions.where, options);
+  private where(options: WhereOptions, include?: IncludeOptions) {
+    let where: WhereOptions;
+    if (include != null) {
+      if (include.where == null) include.where = {};
+      where = include.where;
+    } else {
+      if (this.findOptions.where == null) this.findOptions.where = {};
+      where = this.findOptions.where;
+    }
+    Object.assign(where, options);
   }
 }
 
diff --git a/libs/database/src/lib/entities/organisation.entity.ts b/libs/database/src/lib/entities/organisation.entity.ts
index 2aafce9..f22f39d 100644
--- a/libs/database/src/lib/entities/organisation.entity.ts
+++ b/libs/database/src/lib/entities/organisation.entity.ts
@@ -1,27 +1,7 @@
-import {
-  AllowNull,
-  AutoIncrement,
-  Column,
-  Default,
-  Index,
-  Model,
-  PrimaryKey,
-  Table,
-} from 'sequelize-typescript';
-import {
-  BIGINT,
-  BOOLEAN,
-  DATE,
-  DECIMAL,
-  ENUM,
-  INTEGER,
-  STRING,
-  TEXT,
-  TINYINT,
-  UUID
-} from 'sequelize';
-
-@Table({ tableName: 'organisations', underscored: true })
+import { AllowNull, AutoIncrement, Column, Default, Index, Model, PrimaryKey, Table } from "sequelize-typescript";
+import { BIGINT, BOOLEAN, DATE, DECIMAL, ENUM, INTEGER, STRING, TEXT, TINYINT, UUID } from "sequelize";
+
+@Table({ tableName: "organisations", underscored: true })
 export class Organisation extends Model {
   @PrimaryKey
   @AutoIncrement
@@ -32,7 +12,7 @@ export class Organisation extends Model {
   @Column(UUID)
   uuid: string;
 
-  @Default('draft')
+  @Default("draft")
   @Column(STRING)
   status: string;
 
@@ -44,6 +24,10 @@ export class Organisation extends Model {
   @Column(BOOLEAN)
   private: boolean;
 
+  @Default(false)
+  @Column(BOOLEAN)
+  isTest: boolean;
+
   @AllowNull
   @Column(STRING)
   name: string | null;
@@ -53,11 +37,11 @@ export class Organisation extends Model {
   phone: string | null;
 
   @AllowNull
-  @Column({ type: STRING, field: 'hq_street_1' })
+  @Column({ type: STRING, field: "hq_street_1" })
   hqStreet1: string | null;
 
   @AllowNull
-  @Column({ type: STRING, field: 'hq_street_2' })
+  @Column({ type: STRING, field: "hq_street_2" })
   hqStreet2: string | null;
 
   @AllowNull
@@ -105,7 +89,7 @@ export class Organisation extends Model {
   relevantExperienceYears: number | null;
 
   @AllowNull
-  @Column({ type: INTEGER({ length: 11 }), field: 'trees_grown_3year' })
+  @Column({ type: INTEGER({ length: 11 }), field: "trees_grown_3year" })
   treesGrown3Year: number | null;
 
   @AllowNull
@@ -113,7 +97,7 @@ export class Organisation extends Model {
   treesGrownTotal: number | null;
 
   @AllowNull
-  @Column({ type: DECIMAL(8, 2), field: 'ha_restored_3year' })
+  @Column({ type: DECIMAL(8, 2), field: "ha_restored_3year" })
   haRestored3Year: number | null;
 
   @AllowNull
@@ -129,15 +113,15 @@ export class Organisation extends Model {
   finBudgetCurrentYear: number | null;
 
   @AllowNull
-  @Column({ type: DECIMAL(15, 2), field: 'fin_budget_1year' })
+  @Column({ type: DECIMAL(15, 2), field: "fin_budget_1year" })
   finBudget1Year: number | null;
 
   @AllowNull
-  @Column({ type: DECIMAL(15, 2), field: 'fin_budget_2year' })
+  @Column({ type: DECIMAL(15, 2), field: "fin_budget_2year" })
   finBudget2Year: number | null;
 
   @AllowNull
-  @Column({ type: DECIMAL(15, 2), field: 'fin_budget_3year' })
+  @Column({ type: DECIMAL(15, 2), field: "fin_budget_3year" })
   finBudget3Year: number | null;
 
   @AllowNull
@@ -185,7 +169,7 @@ export class Organisation extends Model {
   youngEmployees: number | null;
 
   @AllowNull
-  @Column({ type: INTEGER({ length: 10 }), field: 'over_35_employees' })
+  @Column({ type: INTEGER({ length: 10 }), field: "over_35_employees" })
   over35Employees: number | null;
 
   @AllowNull
@@ -197,27 +181,27 @@ export class Organisation extends Model {
   communityExperience: string | null;
 
   @AllowNull
-  @Column({ type: INTEGER({ length: 10, unsigned: true }), field: 'total_engaged_community_members_3yr' })
+  @Column({ type: INTEGER({ length: 10, unsigned: true }), field: "total_engaged_community_members_3yr" })
   totalEngagedCommunityMembers3Yr: number | null;
 
   @AllowNull
-  @Column({ type: TINYINT({ length: 4 }), field: 'percent_engaged_women_3yr' })
+  @Column({ type: TINYINT({ length: 4 }), field: "percent_engaged_women_3yr" })
   percentEngagedWomen3Yr: number | null;
 
   @AllowNull
-  @Column({ type: TINYINT({ length: 4 }), field: 'percent_engaged_men_3yr' })
+  @Column({ type: TINYINT({ length: 4 }), field: "percent_engaged_men_3yr" })
   percentEngagedMen3Yr: number | null;
 
   @AllowNull
-  @Column({ type: TINYINT({ length: 4 }), field: 'percent_engaged_under_35_3yr' })
+  @Column({ type: TINYINT({ length: 4 }), field: "percent_engaged_under_35_3yr" })
   percentEngagedUnder353Yr: number | null;
 
   @AllowNull
-  @Column({ type: TINYINT({ length: 4 }), field: 'percent_engaged_over_35_3yr' })
+  @Column({ type: TINYINT({ length: 4 }), field: "percent_engaged_over_35_3yr" })
   percentEngagedOver353Yr: number | null;
 
   @AllowNull
-  @Column({ type: TINYINT({ length: 4 }), field: 'percent_engaged_smallholder_3yr' })
+  @Column({ type: TINYINT({ length: 4 }), field: "percent_engaged_smallholder_3yr" })
   percentEngagedSmallholder3Yr: number | null;
 
   @AllowNull
@@ -249,7 +233,7 @@ export class Organisation extends Model {
   monitoringEvaluationExperience: string | null;
 
   @AllowNull
-  @Column(TEXT('long'))
+  @Column(TEXT("long"))
   fundingHistory: string | null;
 
   @AllowNull
@@ -264,7 +248,7 @@ export class Organisation extends Model {
   @Column(TEXT)
   engagementYouth: string | null;
 
-  @Default('usd')
+  @Default("usd")
   @Column(STRING)
   currency: string;
 
@@ -277,11 +261,11 @@ export class Organisation extends Model {
   district: string | null;
 
   @AllowNull
-  @Column({ type: TEXT, field: 'account_number_1' })
+  @Column({ type: TEXT, field: "account_number_1" })
   accountNumber1: string | null;
 
   @AllowNull
-  @Column({ type: TEXT, field: 'account_number_2' })
+  @Column({ type: TEXT, field: "account_number_2" })
   accountNumber2: string | null;
 
   @AllowNull
@@ -313,31 +297,31 @@ export class Organisation extends Model {
   detailedInterventionTypes: string | null;
 
   @AllowNull
-  @Column({ type: INTEGER({ length: 11 }), field: 'community_members_engaged_3yr' })
+  @Column({ type: INTEGER({ length: 11 }), field: "community_members_engaged_3yr" })
   communityMembersEngaged3yr: number | null;
 
   @AllowNull
-  @Column({ type: INTEGER({ length: 11 }), field: 'community_members_engaged_3yr_women' })
+  @Column({ type: INTEGER({ length: 11 }), field: "community_members_engaged_3yr_women" })
   communityMembersEngaged3yrWomen: number | null;
 
   @AllowNull
-  @Column({ type: INTEGER({ length: 11 }), field: 'community_members_engaged_3yr_men' })
+  @Column({ type: INTEGER({ length: 11 }), field: "community_members_engaged_3yr_men" })
   communityMembersEngaged3yrMen: number | null;
 
   @AllowNull
-  @Column({ type: INTEGER({ length: 11 }), field: 'community_members_engaged_3yr_youth' })
+  @Column({ type: INTEGER({ length: 11 }), field: "community_members_engaged_3yr_youth" })
   communityMembersEngaged3yrYouth: number | null;
 
   @AllowNull
-  @Column({ type: INTEGER({ length: 11 }), field: 'community_members_engaged_3yr_non_youth' })
+  @Column({ type: INTEGER({ length: 11 }), field: "community_members_engaged_3yr_non_youth" })
   communityMembersEngaged3yrNonYouth: number | null;
 
   @AllowNull
-  @Column({ type: INTEGER({ length: 11 }), field: 'community_members_engaged_3yr_smallholder' })
+  @Column({ type: INTEGER({ length: 11 }), field: "community_members_engaged_3yr_smallholder" })
   communityMembersEngaged3yrSmallholder: number | null;
 
   @AllowNull
-  @Column({ type: INTEGER({ length: 11 }), field: 'community_members_engaged_3yr_backward_class' })
+  @Column({ type: INTEGER({ length: 11 }), field: "community_members_engaged_3yr_backward_class" })
   communityMembersEngaged3YrBackwardClass: number | null;
 
   @AllowNull
@@ -385,7 +369,7 @@ export class Organisation extends Model {
   fieldStaffSkills: string | null;
 
   @AllowNull
-  @Column({ type: ENUM, values: ['yes', 'no'] })
+  @Column({ type: ENUM, values: ["yes", "no"] })
   fpcCompany: string | null;
 
   @AllowNull
@@ -434,7 +418,7 @@ export class Organisation extends Model {
 
   // field misspelled intentionally to match the current DB schema
   @AllowNull
-  @Column({ type: TEXT, field: 'growith_stage' })
+  @Column({ type: TEXT, field: "growith_stage" })
   growthStage: string | null;
 
   @AllowNull
diff --git a/libs/database/src/lib/entities/project.entity.ts b/libs/database/src/lib/entities/project.entity.ts
index 9371dca..65ba642 100644
--- a/libs/database/src/lib/entities/project.entity.ts
+++ b/libs/database/src/lib/entities/project.entity.ts
@@ -1,5 +1,5 @@
-import { AutoIncrement, Column, Model, PrimaryKey, Table } from "sequelize-typescript";
-import { BIGINT, STRING } from "sequelize";
+import { AutoIncrement, Column, Default, Model, PrimaryKey, Table } from "sequelize-typescript";
+import { BIGINT, BOOLEAN, STRING } from "sequelize";
 
 // A quick stub to get the information needed for users/me
 @Table({ tableName: "v2_projects", underscored: true, paranoid: true })
@@ -11,4 +11,8 @@ export class Project extends Model {
 
   @Column(STRING)
   frameworkKey: string;
+
+  @Default(false)
+  @Column(BOOLEAN)
+  isTest: boolean;
 }
diff --git a/libs/database/src/lib/entities/site.entity.ts b/libs/database/src/lib/entities/site.entity.ts
index 8ec2dad..accea8a 100644
--- a/libs/database/src/lib/entities/site.entity.ts
+++ b/libs/database/src/lib/entities/site.entity.ts
@@ -1,7 +1,8 @@
-import { AutoIncrement, Column, HasMany, Index, Model, PrimaryKey, Table } from "sequelize-typescript";
+import { AutoIncrement, Column, ForeignKey, HasMany, Index, Model, PrimaryKey, Table } from "sequelize-typescript";
 import { BIGINT, UUID } from "sequelize";
 import { TreeSpecies } from "./tree-species.entity";
 import { SiteReport } from "./site-report.entity";
+import { Project } from "./project.entity";
 
 // A quick stub for the research endpoints
 @Table({ tableName: "v2_sites", underscored: true, paranoid: true })
@@ -15,6 +16,10 @@ export class Site extends Model<Site> {
   @Column(UUID)
   uuid: string;
 
+  @ForeignKey(() => Project)
+  @Column(BIGINT.UNSIGNED)
+  projectId: number;
+
   @HasMany(() => TreeSpecies, {
     foreignKey: "speciesableId",
     scope: { speciesableType: "App\\Models\\V2\\Sites\\Site" }

From 26a86227369ca2bd3cf4ebb676ae2972db64a4c4 Mon Sep 17 00:00:00 2001
From: Nathan Curtis <nathan.curtis@gmail.com>
Date: Mon, 18 Nov 2024 12:51:27 -0800
Subject: [PATCH 03/16] [TM-1452] Implement polygon status filter.

---
 .../src/site-polygons/site-polygons.controller.ts           | 4 ++++
 .../src/site-polygons/site-polygons.service.ts              | 6 ++++++
 2 files changed, 10 insertions(+)

diff --git a/apps/research-service/src/site-polygons/site-polygons.controller.ts b/apps/research-service/src/site-polygons/site-polygons.controller.ts
index 14211b1..526f132 100644
--- a/apps/research-service/src/site-polygons/site-polygons.controller.ts
+++ b/apps/research-service/src/site-polygons/site-polygons.controller.ts
@@ -62,6 +62,10 @@ export class SitePolygonsController {
       await queryBuilder.excludeTestProjects();
     }
 
+    if (query.polygonStatus != null) {
+      queryBuilder.filterPolygonStatuses(query.polygonStatus);
+    }
+
     const document = buildJsonApi({ pagination: true });
     for (const sitePolygon of await queryBuilder.execute()) {
       const geometry = await sitePolygon.loadPolygon();
diff --git a/apps/research-service/src/site-polygons/site-polygons.service.ts b/apps/research-service/src/site-polygons/site-polygons.service.ts
index 3637722..e051e04 100644
--- a/apps/research-service/src/site-polygons/site-polygons.service.ts
+++ b/apps/research-service/src/site-polygons/site-polygons.service.ts
@@ -5,6 +5,7 @@ import { IndicatorDto, ReportingPeriodDto, TreeSpeciesDto } from "./dto/site-pol
 import { INDICATOR_DTOS } from "./dto/indicators.dto";
 import { ModelPropertiesAccessor } from "@nestjs/swagger/dist/services/model-properties-accessor";
 import { pick } from "lodash";
+import { PolygonStatus } from "@terramatch-microservices/database/constants";
 
 export class SitePolygonQueryBuilder {
   private siteJoin: IncludeOptions = {
@@ -36,6 +37,11 @@ export class SitePolygonQueryBuilder {
     return this;
   }
 
+  filterPolygonStatuses(polygonStatuses: PolygonStatus[]) {
+    this.where({ status: { [Op.in]: polygonStatuses } });
+    return this;
+  }
+
   async pageAfter(pageAfter: string) {
     const sitePolygon = await SitePolygon.findOne({ where: { uuid: pageAfter }, attributes: ["id"] });
     if (sitePolygon == null) throw new BadRequestException("pageAfter polygon not found");

From 412ac96aa554554dcf959b6973202bc9419eca07 Mon Sep 17 00:00:00 2001
From: Nathan Curtis <nathan.curtis@gmail.com>
Date: Mon, 18 Nov 2024 13:02:12 -0800
Subject: [PATCH 04/16] [TM-1452] Implement filtering on project UUID

---
 .../src/site-polygons/dto/site-polygon-query.dto.ts         | 5 +++--
 .../src/site-polygons/site-polygons.controller.ts           | 6 +++++-
 .../src/site-polygons/site-polygons.service.ts              | 6 ++++++
 3 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/apps/research-service/src/site-polygons/dto/site-polygon-query.dto.ts b/apps/research-service/src/site-polygons/dto/site-polygon-query.dto.ts
index 47ccf13..797fa8f 100644
--- a/apps/research-service/src/site-polygons/dto/site-polygon-query.dto.ts
+++ b/apps/research-service/src/site-polygons/dto/site-polygon-query.dto.ts
@@ -46,7 +46,7 @@ export class SitePolygonQueryDto {
     name: "projectId[]",
     isArray: true,
     required: false,
-    description: "Filter results by project UUID(s)"
+    description: "Filter results by project UUID(s). If specified, the includeTestProjects param is ignored"
   })
   @IsOptional()
   @IsArray()
@@ -81,7 +81,8 @@ export class SitePolygonQueryDto {
   @ApiProperty({
     required: false,
     default: false,
-    description: "Include polygons for test projects in the results"
+    description:
+      "Include polygons for test projects in the results. If an explicit list of project UUIDs is included in projectId[], this parameter is ignored."
   })
   includeTestProjects?: boolean;
 
diff --git a/apps/research-service/src/site-polygons/site-polygons.controller.ts b/apps/research-service/src/site-polygons/site-polygons.controller.ts
index 526f132..cd35299 100644
--- a/apps/research-service/src/site-polygons/site-polygons.controller.ts
+++ b/apps/research-service/src/site-polygons/site-polygons.controller.ts
@@ -58,7 +58,11 @@ export class SitePolygonsController {
     }
 
     const queryBuilder = await this.sitePolygonService.buildQuery(pageSize, pageAfter);
-    if (query.includeTestProjects !== true) {
+
+    // If projectIds are sent, ignore filtering on project is_test flag.
+    if (query.projectId != null) {
+      await queryBuilder.filterProjectUuids(query.projectId);
+    } else if (query.includeTestProjects !== true) {
       await queryBuilder.excludeTestProjects();
     }
 
diff --git a/apps/research-service/src/site-polygons/site-polygons.service.ts b/apps/research-service/src/site-polygons/site-polygons.service.ts
index e051e04..0ce31a3 100644
--- a/apps/research-service/src/site-polygons/site-polygons.service.ts
+++ b/apps/research-service/src/site-polygons/site-polygons.service.ts
@@ -42,6 +42,12 @@ export class SitePolygonQueryBuilder {
     return this;
   }
 
+  async filterProjectUuids(projectUuids: string[]) {
+    const filterProjects = await Project.findAll({ where: { uuid: { [Op.in]: projectUuids } }, attributes: ["id"] });
+    this.where({ projectId: { [Op.in]: filterProjects.map(({ id }) => id) } }, this.siteJoin);
+    return this;
+  }
+
   async pageAfter(pageAfter: string) {
     const sitePolygon = await SitePolygon.findOne({ where: { uuid: pageAfter }, attributes: ["id"] });
     if (sitePolygon == null) throw new BadRequestException("pageAfter polygon not found");

From 81b53a896372b31325f72d3c9fdf44f4ecb46863 Mon Sep 17 00:00:00 2001
From: Nathan Curtis <nathan.curtis@gmail.com>
Date: Mon, 18 Nov 2024 14:05:51 -0800
Subject: [PATCH 05/16] [TM-1452] Implement lastModifiedDate filter.

---
 .../src/site-polygons/site-polygons.controller.ts            | 4 ++++
 .../src/site-polygons/site-polygons.service.ts               | 5 +++++
 2 files changed, 9 insertions(+)

diff --git a/apps/research-service/src/site-polygons/site-polygons.controller.ts b/apps/research-service/src/site-polygons/site-polygons.controller.ts
index cd35299..98b72f1 100644
--- a/apps/research-service/src/site-polygons/site-polygons.controller.ts
+++ b/apps/research-service/src/site-polygons/site-polygons.controller.ts
@@ -70,6 +70,10 @@ export class SitePolygonsController {
       queryBuilder.filterPolygonStatuses(query.polygonStatus);
     }
 
+    if (query.lastModifiedDate != null) {
+      queryBuilder.modifiedSince(query.lastModifiedDate);
+    }
+
     const document = buildJsonApi({ pagination: true });
     for (const sitePolygon of await queryBuilder.execute()) {
       const geometry = await sitePolygon.loadPolygon();
diff --git a/apps/research-service/src/site-polygons/site-polygons.service.ts b/apps/research-service/src/site-polygons/site-polygons.service.ts
index 0ce31a3..7f2f91b 100644
--- a/apps/research-service/src/site-polygons/site-polygons.service.ts
+++ b/apps/research-service/src/site-polygons/site-polygons.service.ts
@@ -48,6 +48,11 @@ export class SitePolygonQueryBuilder {
     return this;
   }
 
+  modifiedSince(date: Date) {
+    this.where({ updatedAt: { [Op.gte]: date } });
+    return this;
+  }
+
   async pageAfter(pageAfter: string) {
     const sitePolygon = await SitePolygon.findOne({ where: { uuid: pageAfter }, attributes: ["id"] });
     if (sitePolygon == null) throw new BadRequestException("pageAfter polygon not found");

From d4951df967e0bb2803c140d5253bcd29f9b8dba2 Mon Sep 17 00:00:00 2001
From: Nathan Curtis <nathan.curtis@gmail.com>
Date: Tue, 19 Nov 2024 11:13:09 -0800
Subject: [PATCH 06/16] [TM-1452] Implement missing indicator filter.

---
 .../site-polygons/site-polygons.controller.ts |  4 +++
 .../site-polygons/site-polygons.service.ts    | 34 +++++++++++++++++--
 2 files changed, 35 insertions(+), 3 deletions(-)

diff --git a/apps/research-service/src/site-polygons/site-polygons.controller.ts b/apps/research-service/src/site-polygons/site-polygons.controller.ts
index 98b72f1..56f47b0 100644
--- a/apps/research-service/src/site-polygons/site-polygons.controller.ts
+++ b/apps/research-service/src/site-polygons/site-polygons.controller.ts
@@ -74,6 +74,10 @@ export class SitePolygonsController {
       queryBuilder.modifiedSince(query.lastModifiedDate);
     }
 
+    if (query.missingIndicator != null) {
+      queryBuilder.filterMissingIndicator(query.missingIndicator);
+    }
+
     const document = buildJsonApi({ pagination: true });
     for (const sitePolygon of await queryBuilder.execute()) {
       const geometry = await sitePolygon.loadPolygon();
diff --git a/apps/research-service/src/site-polygons/site-polygons.service.ts b/apps/research-service/src/site-polygons/site-polygons.service.ts
index 7f2f91b..bbb9931 100644
--- a/apps/research-service/src/site-polygons/site-polygons.service.ts
+++ b/apps/research-service/src/site-polygons/site-polygons.service.ts
@@ -1,11 +1,24 @@
 import { BadRequestException, Injectable, Type } from "@nestjs/common";
 import { Project, Site, SitePolygon, SiteReport } from "@terramatch-microservices/database/entities";
-import { Attributes, FindOptions, IncludeOptions, Op, WhereOptions } from "sequelize";
+import { Attributes, FindOptions, IncludeOptions, literal, Op, WhereOptions } from "sequelize";
 import { IndicatorDto, ReportingPeriodDto, TreeSpeciesDto } from "./dto/site-polygon.dto";
 import { INDICATOR_DTOS } from "./dto/indicators.dto";
 import { ModelPropertiesAccessor } from "@nestjs/swagger/dist/services/model-properties-accessor";
-import { pick } from "lodash";
-import { PolygonStatus } from "@terramatch-microservices/database/constants";
+import { pick, uniq } from "lodash";
+import { IndicatorSlug, PolygonStatus } from "@terramatch-microservices/database/constants";
+
+const INDICATOR_TABLES: { [Slug in IndicatorSlug]: string } = {
+  treeCover: "indicator_output_tree_cover",
+  treeCoverLoss: "indicator_output_tree_cover_loss",
+  treeCoverLossFires: "indicator_output_tree_cover_loss",
+  restorationByEcoRegion: "indicator_output_hectares",
+  restorationByStrategy: "indicator_output_hectares",
+  restorationByLandUse: "indicator_output_hectares",
+  treeCount: "indicator_output_tree_count",
+  earlyTreeVerification: "indicator_output_tree_count",
+  fieldMonitoring: "indicator_output_field_monitoring",
+  msuCarbon: "indicator_output_msu_carbon"
+};
 
 export class SitePolygonQueryBuilder {
   private siteJoin: IncludeOptions = {
@@ -53,6 +66,21 @@ export class SitePolygonQueryBuilder {
     return this;
   }
 
+  filterMissingIndicator(indicatorSlugs: IndicatorSlug[]) {
+    const literals = uniq(indicatorSlugs).map(slug => {
+      const table = INDICATOR_TABLES[slug];
+      if (table == null) throw new BadRequestException(`Unrecognized indicator slug: ${slug}`);
+
+      return literal(
+        `(SELECT COUNT(*) = 0 from ${table} WHERE indicator_slug = "${slug}" AND site_polygon_id = SitePolygon.id)`
+      );
+    });
+    // Note: If we end up needing to use this [Op.and] trick for another query in this builder, we'll need
+    // to make where() smart enough to merge arrays of literals on that query  member.
+    this.where({ [Op.and]: literals });
+    return this;
+  }
+
   async pageAfter(pageAfter: string) {
     const sitePolygon = await SitePolygon.findOne({ where: { uuid: pageAfter }, attributes: ["id"] });
     if (sitePolygon == null) throw new BadRequestException("pageAfter polygon not found");

From 1884773cf1b71c53220052952f04a0c03770670c Mon Sep 17 00:00:00 2001
From: Nathan Curtis <nathan.curtis@gmail.com>
Date: Tue, 19 Nov 2024 11:26:42 -0800
Subject: [PATCH 07/16] [TM-1452] streamline simple filters.

---
 .../site-polygons/site-polygons.controller.ts | 17 +++-------
 .../site-polygons/site-polygons.service.ts    | 34 ++++++++++---------
 2 files changed, 22 insertions(+), 29 deletions(-)

diff --git a/apps/research-service/src/site-polygons/site-polygons.controller.ts b/apps/research-service/src/site-polygons/site-polygons.controller.ts
index 56f47b0..ed26453 100644
--- a/apps/research-service/src/site-polygons/site-polygons.controller.ts
+++ b/apps/research-service/src/site-polygons/site-polygons.controller.ts
@@ -57,7 +57,10 @@ export class SitePolygonsController {
       throw new BadRequestException("Page size is invalid");
     }
 
-    const queryBuilder = await this.sitePolygonService.buildQuery(pageSize, pageAfter);
+    const queryBuilder = (await this.sitePolygonService.buildQuery(pageSize, pageAfter))
+      .filterPolygonStatuses(query.polygonStatus)
+      .modifiedSince(query.lastModifiedDate)
+      .filterMissingIndicator(query.missingIndicator);
 
     // If projectIds are sent, ignore filtering on project is_test flag.
     if (query.projectId != null) {
@@ -66,18 +69,6 @@ export class SitePolygonsController {
       await queryBuilder.excludeTestProjects();
     }
 
-    if (query.polygonStatus != null) {
-      queryBuilder.filterPolygonStatuses(query.polygonStatus);
-    }
-
-    if (query.lastModifiedDate != null) {
-      queryBuilder.modifiedSince(query.lastModifiedDate);
-    }
-
-    if (query.missingIndicator != null) {
-      queryBuilder.filterMissingIndicator(query.missingIndicator);
-    }
-
     const document = buildJsonApi({ pagination: true });
     for (const sitePolygon of await queryBuilder.execute()) {
       const geometry = await sitePolygon.loadPolygon();
diff --git a/apps/research-service/src/site-polygons/site-polygons.service.ts b/apps/research-service/src/site-polygons/site-polygons.service.ts
index bbb9931..a304c05 100644
--- a/apps/research-service/src/site-polygons/site-polygons.service.ts
+++ b/apps/research-service/src/site-polygons/site-polygons.service.ts
@@ -50,8 +50,8 @@ export class SitePolygonQueryBuilder {
     return this;
   }
 
-  filterPolygonStatuses(polygonStatuses: PolygonStatus[]) {
-    this.where({ status: { [Op.in]: polygonStatuses } });
+  filterPolygonStatuses(polygonStatuses?: PolygonStatus[]) {
+    if (polygonStatuses != null) this.where({ status: { [Op.in]: polygonStatuses } });
     return this;
   }
 
@@ -61,23 +61,25 @@ export class SitePolygonQueryBuilder {
     return this;
   }
 
-  modifiedSince(date: Date) {
-    this.where({ updatedAt: { [Op.gte]: date } });
+  modifiedSince(date?: Date) {
+    if (date != null) this.where({ updatedAt: { [Op.gte]: date } });
     return this;
   }
 
-  filterMissingIndicator(indicatorSlugs: IndicatorSlug[]) {
-    const literals = uniq(indicatorSlugs).map(slug => {
-      const table = INDICATOR_TABLES[slug];
-      if (table == null) throw new BadRequestException(`Unrecognized indicator slug: ${slug}`);
-
-      return literal(
-        `(SELECT COUNT(*) = 0 from ${table} WHERE indicator_slug = "${slug}" AND site_polygon_id = SitePolygon.id)`
-      );
-    });
-    // Note: If we end up needing to use this [Op.and] trick for another query in this builder, we'll need
-    // to make where() smart enough to merge arrays of literals on that query  member.
-    this.where({ [Op.and]: literals });
+  filterMissingIndicator(indicatorSlugs?: IndicatorSlug[]) {
+    if (indicatorSlugs != null) {
+      const literals = uniq(indicatorSlugs).map(slug => {
+        const table = INDICATOR_TABLES[slug];
+        if (table == null) throw new BadRequestException(`Unrecognized indicator slug: ${slug}`);
+
+        return literal(
+          `(SELECT COUNT(*) = 0 from ${table} WHERE indicator_slug = "${slug}" AND site_polygon_id = SitePolygon.id)`
+        );
+      });
+      // Note: If we end up needing to use this [Op.and] trick for another query in this builder, we'll need
+      // to make where() smart enough to merge arrays of literals on that query  member.
+      this.where({ [Op.and]: literals });
+    }
     return this;
   }
 

From 5c0b43475c157ce63cec56cb4d17148dc8f9080d Mon Sep 17 00:00:00 2001
From: Nathan Curtis <nathan.curtis@gmail.com>
Date: Tue, 19 Nov 2024 12:10:43 -0800
Subject: [PATCH 08/16] [TM-1452] Implement boundary polygon filter.

---
 .../dto/site-polygon-query.dto.ts             |  2 +-
 .../site-polygons/site-polygons.controller.ts |  6 +-
 .../site-polygons/site-polygons.service.ts    | 62 +++++++++++++------
 3 files changed, 47 insertions(+), 23 deletions(-)

diff --git a/apps/research-service/src/site-polygons/dto/site-polygon-query.dto.ts b/apps/research-service/src/site-polygons/dto/site-polygon-query.dto.ts
index 797fa8f..7df8404 100644
--- a/apps/research-service/src/site-polygons/dto/site-polygon-query.dto.ts
+++ b/apps/research-service/src/site-polygons/dto/site-polygon-query.dto.ts
@@ -73,7 +73,7 @@ export class SitePolygonQueryDto {
 
   @ApiProperty({
     required: false,
-    description: "Filter results by polygons that are within the boundary of the polygon referenced by this UUID"
+    description: "Filter results by polygons that intersect with the boundary of the polygon referenced by this UUID"
   })
   @IsOptional()
   boundaryPolygon?: string;
diff --git a/apps/research-service/src/site-polygons/site-polygons.controller.ts b/apps/research-service/src/site-polygons/site-polygons.controller.ts
index ed26453..771924f 100644
--- a/apps/research-service/src/site-polygons/site-polygons.controller.ts
+++ b/apps/research-service/src/site-polygons/site-polygons.controller.ts
@@ -58,9 +58,11 @@ export class SitePolygonsController {
     }
 
     const queryBuilder = (await this.sitePolygonService.buildQuery(pageSize, pageAfter))
-      .filterPolygonStatuses(query.polygonStatus)
+      .hasStatuses(query.polygonStatus)
       .modifiedSince(query.lastModifiedDate)
-      .filterMissingIndicator(query.missingIndicator);
+      .isMissingIndicators(query.missingIndicator);
+
+    await queryBuilder.touchesBoundary(query.boundaryPolygon);
 
     // If projectIds are sent, ignore filtering on project is_test flag.
     if (query.projectId != null) {
diff --git a/apps/research-service/src/site-polygons/site-polygons.service.ts b/apps/research-service/src/site-polygons/site-polygons.service.ts
index a304c05..606255a 100644
--- a/apps/research-service/src/site-polygons/site-polygons.service.ts
+++ b/apps/research-service/src/site-polygons/site-polygons.service.ts
@@ -1,6 +1,6 @@
 import { BadRequestException, Injectable, Type } from "@nestjs/common";
-import { Project, Site, SitePolygon, SiteReport } from "@terramatch-microservices/database/entities";
-import { Attributes, FindOptions, IncludeOptions, literal, Op, WhereOptions } from "sequelize";
+import { PolygonGeometry, Project, Site, SitePolygon, SiteReport } from "@terramatch-microservices/database/entities";
+import { Attributes, Filterable, FindOptions, IncludeOptions, literal, Op, WhereOptions } from "sequelize";
 import { IndicatorDto, ReportingPeriodDto, TreeSpeciesDto } from "./dto/site-polygon.dto";
 import { INDICATOR_DTOS } from "./dto/indicators.dto";
 import { ModelPropertiesAccessor } from "@nestjs/swagger/dist/services/model-properties-accessor";
@@ -26,6 +26,10 @@ export class SitePolygonQueryBuilder {
     include: ["treeSpecies", { model: SiteReport, include: ["treeSpecies"] }],
     required: true
   };
+  private polygonJoin: IncludeOptions = {
+    model: PolygonGeometry,
+    required: true
+  };
   private findOptions: FindOptions<Attributes<SitePolygon>> = {
     include: [
       "indicatorsFieldMonitoring",
@@ -34,7 +38,7 @@ export class SitePolygonQueryBuilder {
       "indicatorsTreeCount",
       "indicatorsTreeCover",
       "indicatorsTreeCoverLoss",
-      "polygon",
+      this.polygonJoin,
       this.siteJoin
     ]
   };
@@ -50,23 +54,23 @@ export class SitePolygonQueryBuilder {
     return this;
   }
 
-  filterPolygonStatuses(polygonStatuses?: PolygonStatus[]) {
-    if (polygonStatuses != null) this.where({ status: { [Op.in]: polygonStatuses } });
-    return this;
-  }
-
   async filterProjectUuids(projectUuids: string[]) {
     const filterProjects = await Project.findAll({ where: { uuid: { [Op.in]: projectUuids } }, attributes: ["id"] });
     this.where({ projectId: { [Op.in]: filterProjects.map(({ id }) => id) } }, this.siteJoin);
     return this;
   }
 
+  hasStatuses(polygonStatuses?: PolygonStatus[]) {
+    if (polygonStatuses != null) this.where({ status: { [Op.in]: polygonStatuses } });
+    return this;
+  }
+
   modifiedSince(date?: Date) {
     if (date != null) this.where({ updatedAt: { [Op.gte]: date } });
     return this;
   }
 
-  filterMissingIndicator(indicatorSlugs?: IndicatorSlug[]) {
+  isMissingIndicators(indicatorSlugs?: IndicatorSlug[]) {
     if (indicatorSlugs != null) {
       const literals = uniq(indicatorSlugs).map(slug => {
         const table = INDICATOR_TABLES[slug];
@@ -76,13 +80,31 @@ export class SitePolygonQueryBuilder {
           `(SELECT COUNT(*) = 0 from ${table} WHERE indicator_slug = "${slug}" AND site_polygon_id = SitePolygon.id)`
         );
       });
-      // Note: If we end up needing to use this [Op.and] trick for another query in this builder, we'll need
-      // to make where() smart enough to merge arrays of literals on that query  member.
       this.where({ [Op.and]: literals });
     }
     return this;
   }
 
+  async touchesBoundary(polygonUuid?: string) {
+    if (polygonUuid != null) {
+      // This check isn't strictly necessary for constructing the query, but we do want to throw a useful
+      // error to the caller if the polygonUuid doesn't exist, and simply mixing it into the query won't
+      // do it
+      if ((await PolygonGeometry.count({ where: { uuid: polygonUuid } })) === 0) {
+        throw new BadRequestException(`Unrecognized polygon UUID: ${polygonUuid}`);
+      }
+
+      this.where({
+        [Op.and]: [
+          literal(
+            `(SELECT ST_INTERSECTS(polygon.geom, (SELECT geom FROM polygon_geometry WHERE uuid = "${polygonUuid}")))`
+          )
+        ]
+      });
+    }
+    return this;
+  }
+
   async pageAfter(pageAfter: string) {
     const sitePolygon = await SitePolygon.findOne({ where: { uuid: pageAfter }, attributes: ["id"] });
     if (sitePolygon == null) throw new BadRequestException("pageAfter polygon not found");
@@ -94,16 +116,16 @@ export class SitePolygonQueryBuilder {
     return await SitePolygon.findAll(this.findOptions);
   }
 
-  private where(options: WhereOptions, include?: IncludeOptions) {
-    let where: WhereOptions;
-    if (include != null) {
-      if (include.where == null) include.where = {};
-      where = include.where;
-    } else {
-      if (this.findOptions.where == null) this.findOptions.where = {};
-      where = this.findOptions.where;
+  private where(options: WhereOptions, filterable: Filterable = this.findOptions) {
+    if (filterable.where == null) filterable.where = {};
+
+    const clauses = { ...options };
+    if (clauses[Op.and] != null && filterable.where[Op.and] != null) {
+      // For this builder, we only use arrays of literals with Op.and, so we can simply merge the arrays
+      clauses[Op.and] = [...filterable.where[Op.and], ...clauses[Op.and]];
     }
-    Object.assign(where, options);
+
+    Object.assign(filterable.where, clauses);
   }
 }
 

From cf9f0a8006ad38dd5d10c29d7fe28f466ece578c Mon Sep 17 00:00:00 2001
From: Nathan Curtis <nathan.curtis@gmail.com>
Date: Tue, 19 Nov 2024 13:23:29 -0800
Subject: [PATCH 09/16] [TM-1452] Be picky about attributes needed to greatly
 reduce the query size.

---
 .../site-polygons/site-polygons.service.ts    | 45 +++++++++++++------
 1 file changed, 32 insertions(+), 13 deletions(-)

diff --git a/apps/research-service/src/site-polygons/site-polygons.service.ts b/apps/research-service/src/site-polygons/site-polygons.service.ts
index 606255a..954fb64 100644
--- a/apps/research-service/src/site-polygons/site-polygons.service.ts
+++ b/apps/research-service/src/site-polygons/site-polygons.service.ts
@@ -1,5 +1,18 @@
 import { BadRequestException, Injectable, Type } from "@nestjs/common";
-import { PolygonGeometry, Project, Site, SitePolygon, SiteReport } from "@terramatch-microservices/database/entities";
+import {
+  IndicatorOutputFieldMonitoring,
+  IndicatorOutputHectares,
+  IndicatorOutputMsuCarbon,
+  IndicatorOutputTreeCount,
+  IndicatorOutputTreeCover,
+  IndicatorOutputTreeCoverLoss,
+  PolygonGeometry,
+  Project,
+  Site,
+  SitePolygon,
+  SiteReport,
+  TreeSpecies
+} from "@terramatch-microservices/database/entities";
 import { Attributes, Filterable, FindOptions, IncludeOptions, literal, Op, WhereOptions } from "sequelize";
 import { IndicatorDto, ReportingPeriodDto, TreeSpeciesDto } from "./dto/site-polygon.dto";
 import { INDICATOR_DTOS } from "./dto/indicators.dto";
@@ -20,25 +33,31 @@ const INDICATOR_TABLES: { [Slug in IndicatorSlug]: string } = {
   msuCarbon: "indicator_output_msu_carbon"
 };
 
+const INDICATOR_EXCLUDE_COLUMNS = ["id", "sitePolygonId", "createdAt", "updatedAt", "deletedAt"];
+
 export class SitePolygonQueryBuilder {
   private siteJoin: IncludeOptions = {
     model: Site,
-    include: ["treeSpecies", { model: SiteReport, include: ["treeSpecies"] }],
-    required: true
-  };
-  private polygonJoin: IncludeOptions = {
-    model: PolygonGeometry,
+    include: [
+      { model: TreeSpecies, attributes: ["name", "amount"] },
+      {
+        model: SiteReport,
+        include: [{ model: TreeSpecies, attributes: ["name", "amount"] }],
+        attributes: ["dueAt", "submittedAt"]
+      }
+    ],
+    attributes: ["projectId"],
     required: true
   };
   private findOptions: FindOptions<Attributes<SitePolygon>> = {
     include: [
-      "indicatorsFieldMonitoring",
-      "indicatorsHectares",
-      "indicatorsMsuCarbon",
-      "indicatorsTreeCount",
-      "indicatorsTreeCover",
-      "indicatorsTreeCoverLoss",
-      this.polygonJoin,
+      { model: IndicatorOutputFieldMonitoring, attributes: { exclude: INDICATOR_EXCLUDE_COLUMNS } },
+      { model: IndicatorOutputHectares, attributes: { exclude: INDICATOR_EXCLUDE_COLUMNS } },
+      { model: IndicatorOutputMsuCarbon, attributes: { exclude: INDICATOR_EXCLUDE_COLUMNS } },
+      { model: IndicatorOutputTreeCount, attributes: { exclude: INDICATOR_EXCLUDE_COLUMNS } },
+      { model: IndicatorOutputTreeCover, attributes: { exclude: INDICATOR_EXCLUDE_COLUMNS } },
+      { model: IndicatorOutputTreeCoverLoss, attributes: { exclude: INDICATOR_EXCLUDE_COLUMNS } },
+      { model: PolygonGeometry, attributes: ["polygon"], required: true },
       this.siteJoin
     ]
   };

From 93796bc04cb991327db9d0d810834b496e2c4b21 Mon Sep 17 00:00:00 2001
From: Nathan Curtis <nathan.curtis@gmail.com>
Date: Tue, 19 Nov 2024 16:00:39 -0800
Subject: [PATCH 10/16] [TM-1452] Updated controller specs

---
 .../site-polygons.controller.spec.ts          | 51 +++++++++++++++++--
 .../site-polygons/site-polygons.controller.ts |  2 +-
 2 files changed, 48 insertions(+), 5 deletions(-)

diff --git a/apps/research-service/src/site-polygons/site-polygons.controller.spec.ts b/apps/research-service/src/site-polygons/site-polygons.controller.spec.ts
index 9edb844..9f2d410 100644
--- a/apps/research-service/src/site-polygons/site-polygons.controller.spec.ts
+++ b/apps/research-service/src/site-polygons/site-polygons.controller.spec.ts
@@ -6,12 +6,31 @@ import { PolicyService } from "@terramatch-microservices/common";
 import { BadRequestException, NotImplementedException, UnauthorizedException } from "@nestjs/common";
 import { SitePolygonFactory } from "@terramatch-microservices/database/factories";
 import { Resource } from "@terramatch-microservices/common/util";
+import { SitePolygon } from "@terramatch-microservices/database/entities";
 
 describe("SitePolygonsController", () => {
   let controller: SitePolygonsController;
   let sitePolygonService: DeepMocked<SitePolygonsService>;
   let policyService: DeepMocked<PolicyService>;
 
+  const mockQueryBuilder = (executeResult: SitePolygon[] = []) => {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const builder: any = {
+      execute: jest.fn(),
+      hasStatuses: jest.fn().mockReturnThis(),
+      modifiedSince: jest.fn().mockReturnThis(),
+      isMissingIndicators: jest.fn().mockReturnThis()
+    };
+    builder.touchesBoundary = jest.fn().mockResolvedValue(builder);
+    builder.filterProjectUuids = jest.fn().mockResolvedValue(builder);
+    builder.excludeTestProjects = jest.fn().mockResolvedValue(builder);
+
+    builder.execute.mockResolvedValue(executeResult);
+    sitePolygonService.buildQuery.mockResolvedValue(builder);
+
+    return builder;
+  };
+
   beforeEach(async () => {
     const module: TestingModule = await Test.createTestingModule({
       controllers: [SitePolygonsController],
@@ -49,10 +68,7 @@ describe("SitePolygonsController", () => {
     it("Returns a valid value if the request is valid", async () => {
       policyService.authorize.mockResolvedValue(undefined);
       const sitePolygon = await SitePolygonFactory.create();
-      const Builder = { execute: jest.fn() };
-      Builder.execute.mockResolvedValue([sitePolygon]);
-      // eslint-disable-next-line @typescript-eslint/no-explicit-any
-      sitePolygonService.buildQuery.mockResolvedValue(Builder as any);
+      mockQueryBuilder([sitePolygon]);
       const result = await controller.findMany({});
       expect(result.meta).not.toBe(null);
       expect(result.meta.page.total).toBe(1);
@@ -62,6 +78,33 @@ describe("SitePolygonsController", () => {
       expect(resources.length).toBe(1);
       expect(resources[0].id).toBe(sitePolygon.uuid);
     });
+
+    it("Excludes test projects by default", async () => {
+      policyService.authorize.mockResolvedValue(undefined);
+      const builder = mockQueryBuilder();
+      const result = await controller.findMany({});
+      expect(result.meta.page.total).toBe(0);
+
+      expect(builder.excludeTestProjects).toHaveBeenCalled();
+    });
+
+    it("will either honor projectIds or includeTestProjects", async () => {
+      policyService.authorize.mockResolvedValue(undefined);
+      const builder = mockQueryBuilder();
+
+      await controller.findMany({ projectId: ["asdf"] });
+      expect(builder.filterProjectUuids).toHaveBeenCalledWith(["asdf"]);
+      expect(builder.excludeTestProjects).not.toHaveBeenCalled();
+      builder.filterProjectUuids.mockClear();
+
+      await controller.findMany({ includeTestProjects: true });
+      expect(builder.filterProjectUuids).not.toHaveBeenCalled();
+      expect(builder.excludeTestProjects).not.toHaveBeenCalled();
+
+      await controller.findMany({});
+      expect(builder.filterProjectUuids).not.toHaveBeenCalled();
+      expect(builder.excludeTestProjects).toHaveBeenCalled();
+    });
   });
 
   describe("bulkUpdate", () => {
diff --git a/apps/research-service/src/site-polygons/site-polygons.controller.ts b/apps/research-service/src/site-polygons/site-polygons.controller.ts
index 771924f..8e79651 100644
--- a/apps/research-service/src/site-polygons/site-polygons.controller.ts
+++ b/apps/research-service/src/site-polygons/site-polygons.controller.ts
@@ -48,7 +48,7 @@ export class SitePolygonsController {
   @ApiOperation({ operationId: "sitePolygonsIndex", summary: "Get all site polygons" })
   @JsonApiResponse({ data: { type: SitePolygonDto }, pagination: true })
   @ApiException(() => UnauthorizedException, { description: "Authentication failed." })
-  @ApiException(() => BadRequestException, { description: "Pagination values are invalid." })
+  @ApiException(() => BadRequestException, { description: "One or more query param values is invalid." })
   async findMany(@Query() query: SitePolygonQueryDto): Promise<JsonApiDocument> {
     await this.policyService.authorize("readAll", SitePolygon);
 

From 2a93aee42ee23edd213f3060064279ecf720fe3d Mon Sep 17 00:00:00 2001
From: Nathan Curtis <nathan.curtis@gmail.com>
Date: Tue, 19 Nov 2024 17:18:03 -0800
Subject: [PATCH 11/16] [TM-1452] Unit tests for all filters except
 touchesBoundary

---
 .../site-polygons.service.spec.ts             | 145 ++++++++++++++++++
 .../src/lib/entities/project.entity.ts        |   8 +-
 libs/database/src/lib/factories/index.ts      |   1 +
 .../src/lib/factories/project.factory.ts      |   6 +
 package-lock.json                             |  25 ++-
 package.json                                  |   2 +
 6 files changed, 181 insertions(+), 6 deletions(-)
 create mode 100644 libs/database/src/lib/factories/project.factory.ts

diff --git a/apps/research-service/src/site-polygons/site-polygons.service.spec.ts b/apps/research-service/src/site-polygons/site-polygons.service.spec.ts
index d5eb998..e738404 100644
--- a/apps/research-service/src/site-polygons/site-polygons.service.spec.ts
+++ b/apps/research-service/src/site-polygons/site-polygons.service.spec.ts
@@ -1,3 +1,4 @@
+import FakeTimers from "@sinonjs/fake-timers";
 import { SitePolygonsService } from "./site-polygons.service";
 import { Test, TestingModule } from "@nestjs/testing";
 import {
@@ -7,12 +8,16 @@ import {
   IndicatorOutputTreeCountFactory,
   IndicatorOutputTreeCoverFactory,
   IndicatorOutputTreeCoverLossFactory,
+  ProjectFactory,
+  SiteFactory,
   SitePolygonFactory,
   SiteReportFactory,
   TreeSpeciesFactory
 } from "@terramatch-microservices/database/factories";
 import { Indicator, PolygonGeometry, SitePolygon, TreeSpecies } from "@terramatch-microservices/database/entities";
 import { BadRequestException } from "@nestjs/common";
+import { faker } from "@faker-js/faker";
+import { DateTime } from "luxon";
 
 describe("SitePolygonsService", () => {
   let service: SitePolygonsService;
@@ -121,4 +126,144 @@ describe("SitePolygonsService", () => {
     expect(await service.getEstablishmentTreeSpecies(sitePolygon)).toEqual([]);
     expect(await service.getReportingPeriods(sitePolygon)).toEqual([]);
   });
+
+  it("Should filter out test projects", async () => {
+    await SitePolygon.truncate();
+    const project1 = await ProjectFactory.create({ isTest: true });
+    const site1 = await SiteFactory.create({ projectId: project1.id });
+    const project2 = await ProjectFactory.create();
+    const site2 = await SiteFactory.create({ projectId: project2.id });
+    const poly1 = await SitePolygonFactory.create({ siteUuid: site1.uuid });
+    const poly2 = await SitePolygonFactory.create({ siteUuid: site2.uuid });
+
+    let query = await service.buildQuery(20);
+    await query.excludeTestProjects();
+    let result = await query.execute();
+    expect(result.length).toBe(1);
+    expect(result[0].id).toBe(poly2.id);
+
+    query = await service.buildQuery(20);
+    result = await query.execute();
+    expect(result.length).toBe(2);
+    expect(result.map(({ id }) => id).sort()).toEqual([poly1.id, poly2.id].sort());
+  });
+
+  it("Should only include given projects", async () => {
+    await SitePolygon.truncate();
+    const project = await ProjectFactory.create();
+    const site = await SiteFactory.create({ projectId: project.id });
+    const poly1 = await SitePolygonFactory.create({ siteUuid: site.uuid });
+    const poly2 = await SitePolygonFactory.create();
+
+    let query = await service.buildQuery(20);
+    await query.filterProjectUuids([project.uuid]);
+    let result = await query.execute();
+    expect(result.length).toBe(1);
+    expect(result[0].id).toBe(poly1.id);
+
+    query = await service.buildQuery(20);
+    result = await query.execute();
+    expect(result.length).toBe(2);
+    expect(result.map(({ id }) => id).sort()).toEqual([poly1.id, poly2.id].sort());
+  });
+
+  it("should only include polys with the given statuses", async () => {
+    await SitePolygon.truncate();
+    const draftPoly = await SitePolygonFactory.create({ status: "draft" });
+    const submittedPoly = await SitePolygonFactory.create({ status: "submitted" });
+    const approvedPoly = await SitePolygonFactory.create({ status: "approved" });
+
+    let query = (await service.buildQuery(20)).hasStatuses(["draft"]);
+    let result = await query.execute();
+    expect(result.length).toBe(1);
+    expect(result[0].id).toBe(draftPoly.id);
+
+    query = (await service.buildQuery(20)).hasStatuses(["draft", "approved"]);
+    result = await query.execute();
+    expect(result.length).toBe(2);
+    expect(result.map(({ id }) => id).sort()).toEqual([draftPoly.id, approvedPoly.id].sort());
+
+    query = await service.buildQuery(20);
+    result = await query.execute();
+    expect(result.length).toBe(3);
+    expect(result.map(({ id }) => id).sort()).toEqual([draftPoly.id, submittedPoly.id, approvedPoly.id].sort());
+  });
+
+  it("should only return polys updated since the given date", async () => {
+    // sequelize doesn't support manually setting createdAt or updatedAt, so we have to mess with the
+    // system clock for this test.
+    const clock = FakeTimers.install({ shouldAdvanceTime: true });
+    try {
+      await SitePolygon.truncate();
+      const oldDate = faker.date.past({ years: 1 });
+      const newDate = faker.date.recent();
+      clock.setSystemTime(oldDate);
+      const poly1 = await SitePolygonFactory.create({ status: "draft" });
+      clock.setSystemTime(newDate);
+      const poly2 = await SitePolygonFactory.create();
+
+      let query = (await service.buildQuery(20)).modifiedSince(
+        DateTime.fromJSDate(oldDate).plus({ days: 5 }).toJSDate()
+      );
+      let result = await query.execute();
+      expect(result.length).toBe(1);
+      expect(result[0].id).toBe(poly2.id);
+
+      const updateDate = DateTime.fromJSDate(newDate).plus({ days: 1 }).toJSDate();
+      clock.setSystemTime(updateDate);
+      await poly1.update({ status: "submitted" });
+      // The SQL query uses greater than or equal, but in order to get around weirdness with
+      // truncated date precision, we test with a slightly older date time.
+      query = (await service.buildQuery(20)).modifiedSince(
+        DateTime.fromJSDate(updateDate).minus({ minutes: 1 }).toJSDate()
+      );
+      result = await query.execute();
+      expect(result.length).toBe(1);
+      expect(result[0].id).toBe(poly1.id);
+
+      query = await service.buildQuery(20);
+      result = await query.execute();
+      expect(result.length).toBe(2);
+      expect(result.map(({ id }) => id).sort()).toEqual([poly1.id, poly2.id].sort());
+    } finally {
+      clock.uninstall();
+    }
+  });
+
+  it("should only return polys missing the given indicators", async () => {
+    await SitePolygon.truncate();
+    const poly1 = await SitePolygonFactory.create();
+    await IndicatorOutputFieldMonitoringFactory.create({ sitePolygonId: poly1.id });
+    await IndicatorOutputHectaresFactory.create({ sitePolygonId: poly1.id, indicatorSlug: "restorationByLandUse" });
+    const poly2 = await SitePolygonFactory.create();
+    await IndicatorOutputMsuCarbonFactory.create({ sitePolygonId: poly2.id });
+    await IndicatorOutputHectaresFactory.create({ sitePolygonId: poly2.id, indicatorSlug: "restorationByLandUse" });
+    const poly3 = await SitePolygonFactory.create();
+    await IndicatorOutputHectaresFactory.create({ sitePolygonId: poly3.id, indicatorSlug: "restorationByStrategy" });
+
+    let query = (await service.buildQuery(20)).isMissingIndicators(["fieldMonitoring"]);
+    let result = await query.execute();
+    expect(result.length).toBe(2);
+    expect(result.map(({ id }) => id).sort()).toEqual([poly2.id, poly3.id].sort());
+
+    query = (await service.buildQuery(20)).isMissingIndicators(["restorationByLandUse"]);
+    result = await query.execute();
+    expect(result.length).toBe(1);
+    expect(result[0].id).toBe(poly3.id);
+
+    query = (await service.buildQuery(20)).isMissingIndicators(["restorationByStrategy", "msuCarbon"]);
+    result = await query.execute();
+    expect(result.length).toBe(1);
+    expect(result[0].id).toBe(poly1.id);
+
+    query = (await service.buildQuery(20)).isMissingIndicators(["restorationByEcoRegion"]);
+    result = await query.execute();
+    expect(result.length).toBe(3);
+    expect(result.map(({ id }) => id).sort()).toEqual([poly1.id, poly2.id, poly3.id].sort());
+
+    query = await service.buildQuery(20);
+    result = await query.execute();
+    expect(result.length).toBe(3);
+    expect(result.map(({ id }) => id).sort()).toEqual([poly1.id, poly2.id, poly3.id].sort());
+  });
 });
diff --git a/libs/database/src/lib/entities/project.entity.ts b/libs/database/src/lib/entities/project.entity.ts
index 65ba642..05ec6e3 100644
--- a/libs/database/src/lib/entities/project.entity.ts
+++ b/libs/database/src/lib/entities/project.entity.ts
@@ -1,5 +1,5 @@
-import { AutoIncrement, Column, Default, Model, PrimaryKey, Table } from "sequelize-typescript";
-import { BIGINT, BOOLEAN, STRING } from "sequelize";
+import { AutoIncrement, Column, Default, Index, Model, PrimaryKey, Table } from "sequelize-typescript";
+import { BIGINT, BOOLEAN, STRING, UUID } from "sequelize";
 
 // A quick stub to get the information needed for users/me
 @Table({ tableName: "v2_projects", underscored: true, paranoid: true })
@@ -9,6 +9,10 @@ export class Project extends Model {
   @Column(BIGINT.UNSIGNED)
   override id: number;
 
+  @Index
+  @Column(UUID)
+  uuid: string;
+
   @Column(STRING)
   frameworkKey: string;
 
diff --git a/libs/database/src/lib/factories/index.ts b/libs/database/src/lib/factories/index.ts
index 198408d..a592bc8 100644
--- a/libs/database/src/lib/factories/index.ts
+++ b/libs/database/src/lib/factories/index.ts
@@ -7,6 +7,7 @@ export * from "./indicator-output-tree-cover.factory";
 export * from "./indicator-output-tree-cover-loss.factory";
 export * from "./organisation.factory";
 export * from "./polygon-geometry.factory";
+export * from "./project.factory";
 export * from "./site.factory";
 export * from "./site-polygon.factory";
 export * from "./site-report.factory";
diff --git a/libs/database/src/lib/factories/project.factory.ts b/libs/database/src/lib/factories/project.factory.ts
new file mode 100644
index 0000000..a77443b
--- /dev/null
+++ b/libs/database/src/lib/factories/project.factory.ts
@@ -0,0 +1,6 @@
+import { FactoryGirl } from "factory-girl-ts";
+import { Project } from "../entities";
+
+export const ProjectFactory = FactoryGirl.define(Project, async () => ({
+  uuid: crypto.randomUUID()
+}));
diff --git a/package-lock.json b/package-lock.json
index fbab6f8..68afb55 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -49,6 +49,7 @@
         "@nx/web": "19.7.2",
         "@nx/webpack": "19.7.2",
         "@nx/workspace": "19.7.2",
+        "@sinonjs/fake-timers": "^13.0.5",
         "@swc-node/register": "~1.9.1",
         "@swc/core": "~1.5.7",
         "@swc/helpers": "~0.5.11",
@@ -58,6 +59,7 @@
         "@types/lodash": "^4.17.13",
         "@types/luxon": "^3.4.2",
         "@types/node": "~18.16.9",
+        "@types/sinonjs__fake-timers": "^8.1.5",
         "@types/supertest": "^6.0.2",
         "@types/validator": "^13.12.2",
         "@typescript-eslint/eslint-plugin": "^7.16.0",
@@ -2515,6 +2517,15 @@
         "node": "^14.15.0 || ^16.10.0 || >=18.0.0"
       }
     },
+    "node_modules/@jest/fake-timers/node_modules/@sinonjs/fake-timers": {
+      "version": "10.3.0",
+      "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-10.3.0.tgz",
+      "integrity": "sha512-V4BG07kuYSUkTCSBHG8G8TNhM+F19jXFWnQtzj+we8DrkpSBCee9Z3Ms8yiGer/dlmhe35/Xdgyo3/0rQKg7YA==",
+      "dev": true,
+      "dependencies": {
+        "@sinonjs/commons": "^3.0.0"
+      }
+    },
     "node_modules/@jest/globals": {
       "version": "29.7.0",
       "resolved": "https://registry.npmjs.org/@jest/globals/-/globals-29.7.0.tgz",
@@ -4145,12 +4156,12 @@
       }
     },
     "node_modules/@sinonjs/fake-timers": {
-      "version": "10.3.0",
-      "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-10.3.0.tgz",
-      "integrity": "sha512-V4BG07kuYSUkTCSBHG8G8TNhM+F19jXFWnQtzj+we8DrkpSBCee9Z3Ms8yiGer/dlmhe35/Xdgyo3/0rQKg7YA==",
+      "version": "13.0.5",
+      "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-13.0.5.tgz",
+      "integrity": "sha512-36/hTbH2uaWuGVERyC6da9YwGWnzUZXuPro/F2LfsdOsLnCojz/iSH8MxUt/FD2S5XBSVPhmArFUXcpCQ2Hkiw==",
       "dev": true,
       "dependencies": {
-        "@sinonjs/commons": "^3.0.0"
+        "@sinonjs/commons": "^3.0.1"
       }
     },
     "node_modules/@swc-node/core": {
@@ -4794,6 +4805,12 @@
         "@types/send": "*"
       }
     },
+    "node_modules/@types/sinonjs__fake-timers": {
+      "version": "8.1.5",
+      "resolved": "https://registry.npmjs.org/@types/sinonjs__fake-timers/-/sinonjs__fake-timers-8.1.5.tgz",
+      "integrity": "sha512-mQkU2jY8jJEF7YHjHvsQO8+3ughTL1mcnn96igfhONmR+fUPSKIkefQYpSe8bsly2Ep7oQbn/6VG5/9/0qcArQ==",
+      "dev": true
+    },
     "node_modules/@types/sockjs": {
       "version": "0.3.36",
       "resolved": "https://registry.npmjs.org/@types/sockjs/-/sockjs-0.3.36.tgz",
diff --git a/package.json b/package.json
index 4e169e9..ae4930c 100644
--- a/package.json
+++ b/package.json
@@ -49,6 +49,7 @@
     "@nx/web": "19.7.2",
     "@nx/webpack": "19.7.2",
     "@nx/workspace": "19.7.2",
+    "@sinonjs/fake-timers": "^13.0.5",
     "@swc-node/register": "~1.9.1",
     "@swc/core": "~1.5.7",
     "@swc/helpers": "~0.5.11",
@@ -58,6 +59,7 @@
     "@types/lodash": "^4.17.13",
     "@types/luxon": "^3.4.2",
     "@types/node": "~18.16.9",
+    "@types/sinonjs__fake-timers": "^8.1.5",
     "@types/supertest": "^6.0.2",
     "@types/validator": "^13.12.2",
     "@typescript-eslint/eslint-plugin": "^7.16.0",

From 40ede90a416fada257c1f93782e73bdb757ec11d Mon Sep 17 00:00:00 2001
From: Nathan Curtis <nathan.curtis@gmail.com>
Date: Tue, 19 Nov 2024 21:28:06 -0800
Subject: [PATCH 12/16] [TM-1452] touches boundary and multiple filters tests.

---
 .../site-polygons.controller.spec.ts          |  4 +-
 .../site-polygons.service.spec.ts             | 67 ++++++++++++++++++-
 .../lib/factories/polygon-geometry.factory.ts |  2 +-
 3 files changed, 68 insertions(+), 5 deletions(-)

diff --git a/apps/research-service/src/site-polygons/site-polygons.controller.spec.ts b/apps/research-service/src/site-polygons/site-polygons.controller.spec.ts
index 9f2d410..6d0005b 100644
--- a/apps/research-service/src/site-polygons/site-polygons.controller.spec.ts
+++ b/apps/research-service/src/site-polygons/site-polygons.controller.spec.ts
@@ -4,9 +4,9 @@ import { createMock, DeepMocked } from "@golevelup/ts-jest";
 import { Test, TestingModule } from "@nestjs/testing";
 import { PolicyService } from "@terramatch-microservices/common";
 import { BadRequestException, NotImplementedException, UnauthorizedException } from "@nestjs/common";
-import { SitePolygonFactory } from "@terramatch-microservices/database/factories";
 import { Resource } from "@terramatch-microservices/common/util";
 import { SitePolygon } from "@terramatch-microservices/database/entities";
+import { SitePolygonFactory } from "@terramatch-microservices/database/factories";
 
 describe("SitePolygonsController", () => {
   let controller: SitePolygonsController;
@@ -67,7 +67,7 @@ describe("SitePolygonsController", () => {
 
     it("Returns a valid value if the request is valid", async () => {
       policyService.authorize.mockResolvedValue(undefined);
-      const sitePolygon = await SitePolygonFactory.create();
+      const sitePolygon = await SitePolygonFactory.build();
       mockQueryBuilder([sitePolygon]);
       const result = await controller.findMany({});
       expect(result.meta).not.toBe(null);
diff --git a/apps/research-service/src/site-polygons/site-polygons.service.spec.ts b/apps/research-service/src/site-polygons/site-polygons.service.spec.ts
index e738404..7721ec2 100644
--- a/apps/research-service/src/site-polygons/site-polygons.service.spec.ts
+++ b/apps/research-service/src/site-polygons/site-polygons.service.spec.ts
@@ -8,6 +8,8 @@ import {
   IndicatorOutputTreeCountFactory,
   IndicatorOutputTreeCoverFactory,
   IndicatorOutputTreeCoverLossFactory,
+  POLYGON,
+  PolygonGeometryFactory,
   ProjectFactory,
   SiteFactory,
   SitePolygonFactory,
@@ -18,6 +20,7 @@ import { Indicator, PolygonGeometry, SitePolygon, TreeSpecies } from "@terramatc
 import { BadRequestException } from "@nestjs/common";
 import { faker } from "@faker-js/faker";
 import { DateTime } from "luxon";
+import { IndicatorSlug } from "@terramatch-microservices/database/constants";
 
 describe("SitePolygonsService", () => {
   let service: SitePolygonsService;
@@ -117,8 +120,8 @@ describe("SitePolygonsService", () => {
     expect(result.length).toBe(14);
   });
 
-  it("Should throw when pageAfter polygon not found", () => {
-    expect(service.buildQuery(20, "asdfasdf")).rejects.toThrow(BadRequestException);
+  it("Should throw when pageAfter polygon not found", async () => {
+    await expect(service.buildQuery(20, "asdfasdf")).rejects.toThrow(BadRequestException);
   });
 
   it("Should return empty arrays from utility methods if no associated records exist", async () => {
@@ -266,4 +269,64 @@ describe("SitePolygonsService", () => {
     expect(result.length).toBe(3);
     expect(result.map(({ id }) => id).sort()).toEqual([poly1.id, poly2.id, poly3.id].sort());
   });
+
+  it("throws when an indicator slug is invalid", async () => {
+    const query = await service.buildQuery(20);
+    expect(() => query.isMissingIndicators(["foo" as IndicatorSlug])).toThrow(BadRequestException);
+  });
+
+  it("filters polygons by boundary polygon", async () => {
+    await SitePolygon.truncate();
+    await PolygonGeometry.truncate();
+    const sitePoly1 = await SitePolygonFactory.create();
+    const poly2 = await PolygonGeometryFactory.create({
+      polygon: { ...POLYGON, coordinates: [POLYGON.coordinates[0].map(([lat, lng]) => [lat + 5, lng + 5])] }
+    });
+    const sitePoly2 = await SitePolygonFactory.create({ polygonUuid: poly2.uuid });
+
+    let query = await service.buildQuery(20);
+    await query.touchesBoundary(poly2.uuid);
+    let result = await query.execute();
+    expect(result.length).toBe(1);
+    expect(result[0].id).toBe(sitePoly2.id);
+
+    query = await service.buildQuery(20);
+    result = await query.execute();
+    expect(result.length).toBe(2);
+    expect(result.map(({ id }) => id).sort()).toEqual([sitePoly1.id, sitePoly2.id].sort());
+  });
+
+  it("throws when a boundary poly uuid doesn't exist", async () => {
+    const query = await service.buildQuery(20);
+    await expect(query.touchesBoundary("asdf")).rejects.toThrow(BadRequestException);
+  });
+
+  it("Can apply multiple filter types at once", async () => {
+    await SitePolygon.truncate();
+    const project1 = await ProjectFactory.create({ isTest: true });
+    const site1 = await SiteFactory.create({ projectId: project1.id });
+    const project2 = await ProjectFactory.create();
+    const site2 = await SiteFactory.create({ projectId: project2.id });
+    const draftPoly1 = await SitePolygonFactory.create({ siteUuid: site1.uuid, status: "draft" });
+    await IndicatorOutputHectaresFactory.create({
+      sitePolygonId: draftPoly1.id,
+      indicatorSlug: "restorationByStrategy"
+    });
+    const draftPoly2 = await SitePolygonFactory.create({ siteUuid: site2.uuid, status: "draft" });
+    await SitePolygonFactory.create({ siteUuid: site1.uuid, status: "approved" });
+    const approvedPoly2 = await SitePolygonFactory.create({ siteUuid: site2.uuid, status: "approved" });
+    await IndicatorOutputHectaresFactory.create({
+      sitePolygonId: approvedPoly2.id,
+      indicatorSlug: "restorationByStrategy"
+    });
+
+    const query = (await service.buildQuery(20))
+      .isMissingIndicators(["restorationByStrategy"])
+      .hasStatuses(["draft", "approved"]);
+    await query.filterProjectUuids([project2.uuid]);
+    await query.touchesBoundary(approvedPoly2.polygonUuid);
+    const result = await query.execute();
+    expect(result.length).toBe(1);
+    expect(result[0].id).toBe(draftPoly2.id);
+  });
 });
diff --git a/libs/database/src/lib/factories/polygon-geometry.factory.ts b/libs/database/src/lib/factories/polygon-geometry.factory.ts
index 4228530..268fe3c 100644
--- a/libs/database/src/lib/factories/polygon-geometry.factory.ts
+++ b/libs/database/src/lib/factories/polygon-geometry.factory.ts
@@ -3,7 +3,7 @@ import { PolygonGeometry } from "../entities";
 import { UserFactory } from "./user.factory";
 
 // The shortest polygon defined in the prod DB as of the writing of this test.
-const POLYGON = {
+export const POLYGON = {
   type: "Polygon",
   coordinates: [
     [

From a5aa995eb859026beec9d3a6f54f61b4aa499905 Mon Sep 17 00:00:00 2001
From: Nathan Curtis <nathan.curtis@gmail.com>
Date: Wed, 20 Nov 2024 09:58:33 -0800
Subject: [PATCH 13/16] [TM-1452] Sync database by running one isolated test
 first.

---
 .github/workflows/pull-request.yml            | 20 ++++++++---
 docker-compose.yml                            |  3 +-
 docker/test-connection.sh                     | 10 ++++++
 jest.config.ts                                |  4 +--
 jest.preset.js                                |  8 ++---
 jest/setup-jest.ts                            | 26 ++++++++++++++
 .../src/lib/policies/user.policy.spec.ts      | 35 ++++++++++---------
 libs/database/jest.config.ts                  | 20 +++++++----
 libs/database/src/lib/database.module.spec.ts |  9 +++++
 .../src/lib/entities/delayed-job.entity.ts    | 19 +++-------
 .../src/lib/entities/framework-user.entity.ts | 12 +++----
 .../src/lib/entities/framework.entity.ts      |  8 ++---
 ...ndicator-output-field-monitoring.entity.ts |  2 +-
 .../indicator-output-hectares.entity.ts       |  2 +-
 .../indicator-output-msu-carbon.entity.ts     |  2 +-
 .../indicator-output-tree-count.entity.ts     |  2 +-
 ...indicator-output-tree-cover-loss.entity.ts |  2 +-
 .../indicator-output-tree-cover.entity.ts     |  2 +-
 .../src/lib/entities/model-has-role.entity.ts | 10 +++---
 .../lib/entities/organisation-user.entity.ts  | 12 +++----
 .../src/lib/entities/organisation.entity.ts   |  2 +-
 .../src/lib/entities/permission.entity.ts     | 12 +++----
 .../src/lib/entities/point-geometry.entity.ts |  2 +-
 .../lib/entities/polygon-geometry.entity.ts   |  2 +-
 .../src/lib/entities/project-user.entity.ts   | 20 ++++-------
 .../src/lib/entities/project.entity.ts        |  2 +-
 libs/database/src/lib/entities/role.entity.ts | 14 +++-----
 .../src/lib/entities/site-polygon.entity.ts   |  2 +-
 libs/database/src/lib/entities/user.entity.ts |  2 +-
 libs/database/tsconfig.json                   |  3 +-
 setup-jest.ts                                 | 26 --------------
 31 files changed, 156 insertions(+), 139 deletions(-)
 create mode 100755 docker/test-connection.sh
 create mode 100644 jest/setup-jest.ts
 create mode 100644 libs/database/src/lib/database.module.spec.ts
 delete mode 100644 setup-jest.ts

diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml
index b6bc319..425ea18 100644
--- a/.github/workflows/pull-request.yml
+++ b/.github/workflows/pull-request.yml
@@ -18,6 +18,10 @@ jobs:
         with:
           fetch-depth: 0
 
+      - uses: KengoTODA/actions-setup-docker-compose@v1
+        with:
+          version: '2.29.1'
+
       # This enables task distribution via Nx Cloud
       # Run this command as early as possible, before dependencies are installed
       # Learn more at https://nx.dev/ci/reference/nx-cloud-cli#npx-nxcloud-startcirun
@@ -33,10 +37,16 @@ jobs:
 
       - run: NX_CLOUD_DISTRIBUTED_EXECUTION=false npx nx run-many -t lint build
 
-      - uses: KengoTODA/actions-setup-docker-compose@v1
-        with:
-          version: '2.29.1'
+      - name: Bring up DB Docker Container
+        run: |
+          docker-compose up -d
+          ./docker/test-connection.sh
 
-      - run: docker-compose up -d
+      # First run just the small database test to get the test database synced to the current schema
+      # in a clean way. For some reason, the `run-many` is necessary here. If this line simply uses
+      # nx test database, the connection to the DB gets cut off before the sync is complete.
+      - name: Sync DB Schema
+        run: NX_CLOUD_DISTRIBUTED_EXECUTION=false npx nx test database
 
-      - run: NX_CLOUD_DISTRIBUTED_EXECUTION=false npx nx run-many -t test --coverage --passWithNoTests
+      - name: Test all
+        run: NX_CLOUD_DISTRIBUTED_EXECUTION=false npx nx run-many -t test --coverage --passWithNoTests
diff --git a/docker-compose.yml b/docker-compose.yml
index 3f99ac5..6a17344 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,4 +1,3 @@
-version: "3"
 services:
   mariadb:
     build:
@@ -13,3 +12,5 @@ services:
       MYSQL_DATABASE: "terramatch_microservices_test"
       MYSQL_USER: "wri"
       MYSQL_PASSWORD: "wri"
+networks:
+  default:
diff --git a/docker/test-connection.sh b/docker/test-connection.sh
new file mode 100755
index 0000000..d326655
--- /dev/null
+++ b/docker/test-connection.sh
@@ -0,0 +1,10 @@
+#!/bin/sh
+
+echo "Waiting for MariaDB..."
+until docker-compose exec mariadb mysqladmin ping -hlocalhost -u root -proot --silent
+do
+  echo "MariaDB is not ready will retry in 5..."
+  sleep 5
+done
+
+echo "MariaDB is ready"
diff --git a/jest.config.ts b/jest.config.ts
index 6b3f2d6..7d35575 100644
--- a/jest.config.ts
+++ b/jest.config.ts
@@ -1,5 +1,5 @@
-import { getJestProjectsAsync } from '@nx/jest';
+import { getJestProjectsAsync } from "@nx/jest";
 
 export default async () => ({
-  projects: await getJestProjectsAsync(),
+  projects: await getJestProjectsAsync()
 });
diff --git a/jest.preset.js b/jest.preset.js
index ee18844..4d5b09c 100644
--- a/jest.preset.js
+++ b/jest.preset.js
@@ -1,4 +1,4 @@
-const nxPreset = require('@nx/jest/preset').default;
+const nxPreset = require("@nx/jest/preset").default;
 
 module.exports = {
   ...nxPreset,
@@ -8,9 +8,9 @@ module.exports = {
       branches: 85,
       functions: 95,
       lines: 95,
-      statements: 95,
+      statements: 95
     }
   },
 
-  setupFilesAfterEnv: ['./setup-jest.ts'],
-}
+  setupFilesAfterEnv: ["./jest/setup-jest.ts"]
+};
diff --git a/jest/setup-jest.ts b/jest/setup-jest.ts
new file mode 100644
index 0000000..0c96228
--- /dev/null
+++ b/jest/setup-jest.ts
@@ -0,0 +1,26 @@
+import { Sequelize } from "sequelize-typescript";
+import { FactoryGirl, SequelizeAdapter } from "factory-girl-ts";
+import * as Entities from "@terramatch-microservices/database/entities";
+
+let sequelize: Sequelize;
+
+beforeAll(async () => {
+  // To create this database, run the ./setup-test-database.sh script.
+  sequelize = new Sequelize({
+    dialect: "mariadb",
+    host: "localhost",
+    port: 3360,
+    username: "wri",
+    password: "wri",
+    database: "terramatch_microservices_test",
+    models: Object.values(Entities),
+    logging: false
+  });
+
+  await sequelize.sync();
+  FactoryGirl.setAdapter(new SequelizeAdapter());
+});
+
+afterAll(async () => {
+  await sequelize.close();
+});
diff --git a/libs/common/src/lib/policies/user.policy.spec.ts b/libs/common/src/lib/policies/user.policy.spec.ts
index 695c398..3ad4889 100644
--- a/libs/common/src/lib/policies/user.policy.spec.ts
+++ b/libs/common/src/lib/policies/user.policy.spec.ts
@@ -1,15 +1,16 @@
-import { Test, TestingModule } from '@nestjs/testing';
-import { PolicyService } from './policy.service';
-import { mockPermissions, mockUserId } from './policy.service.spec';
-import { User } from '@terramatch-microservices/database/entities';
-import { UnauthorizedException } from '@nestjs/common';
+import { Test, TestingModule } from "@nestjs/testing";
+import { PolicyService } from "./policy.service";
+import { mockPermissions, mockUserId } from "./policy.service.spec";
+import { User } from "@terramatch-microservices/database/entities";
+import { UnauthorizedException } from "@nestjs/common";
+import { UserFactory } from "@terramatch-microservices/database/factories";
 
-describe('UserPolicy', () => {
+describe("UserPolicy", () => {
   let service: PolicyService;
 
   beforeEach(async () => {
     const module: TestingModule = await Test.createTestingModule({
-      providers: [PolicyService],
+      providers: [PolicyService]
     }).compile();
 
     service = module.get<PolicyService>(PolicyService);
@@ -19,21 +20,21 @@ describe('UserPolicy', () => {
     jest.restoreAllMocks();
   });
 
-  it('allows reading any user as admin', async () => {
+  it("allows reading any user as admin", async () => {
     mockUserId(123);
-    mockPermissions('users-manage');
-    await expect(service.authorize('read', new User())).resolves.toBeUndefined();
-  })
+    mockPermissions("users-manage");
+    await expect(service.authorize("read", new User())).resolves.toBeUndefined();
+  });
 
-  it('disallows reading other users as non-admin', async () => {
+  it("disallows reading other users as non-admin", async () => {
     mockUserId(123);
     mockPermissions();
-    await expect(service.authorize('read', new User())).rejects.toThrow(UnauthorizedException);
-  })
+    await expect(service.authorize("read", new User())).rejects.toThrow(UnauthorizedException);
+  });
 
-  it('allows reading own user as non-admin', async () => {
+  it("allows reading own user as non-admin", async () => {
     mockUserId(123);
     mockPermissions();
-    await expect(service.authorize('read', new User({ id: 123 }))).resolves.toBeUndefined();
-  })
+    await expect(service.authorize("read", await UserFactory.build({ id: 123 }))).resolves.toBeUndefined();
+  });
 });
diff --git a/libs/database/jest.config.ts b/libs/database/jest.config.ts
index 6fef25b..76d3ee0 100644
--- a/libs/database/jest.config.ts
+++ b/libs/database/jest.config.ts
@@ -1,11 +1,19 @@
 /* eslint-disable */
 export default {
-  displayName: 'database',
-  preset: '../../jest.preset.js',
-  testEnvironment: 'node',
+  displayName: "database",
+  preset: "../../jest.preset.js",
+  testEnvironment: "node",
   transform: {
-    '^.+\\.[tj]s$': ['ts-jest', { tsconfig: '<rootDir>/tsconfig.spec.json' }],
+    "^.+\\.[tj]s$": ["ts-jest", { tsconfig: "<rootDir>/tsconfig.spec.json" }]
   },
-  moduleFileExtensions: ['ts', 'js', 'html'],
-  coverageDirectory: '../../coverage/libs/database',
+  moduleFileExtensions: ["ts", "js", "html"],
+  coverageDirectory: "../../coverage/libs/database",
+  coverageThreshold: {
+    global: {
+      branches: 0,
+      functions: 0,
+      lines: 0,
+      statements: 0
+    }
+  }
 };
diff --git a/libs/database/src/lib/database.module.spec.ts b/libs/database/src/lib/database.module.spec.ts
new file mode 100644
index 0000000..adbbf99
--- /dev/null
+++ b/libs/database/src/lib/database.module.spec.ts
@@ -0,0 +1,9 @@
+import * as Entities from "./entities";
+
+describe("DatabaseModule", () => {
+  it("Successfully syncs the database schema", async () => {
+    for (const Entity of Object.values(Entities)) {
+      await expect(Entity.sequelize?.getQueryInterface().tableExists(Entity.tableName)).resolves.toBe(true);
+    }
+  });
+});
diff --git a/libs/database/src/lib/entities/delayed-job.entity.ts b/libs/database/src/lib/entities/delayed-job.entity.ts
index 147dd95..5ced0ad 100644
--- a/libs/database/src/lib/entities/delayed-job.entity.ts
+++ b/libs/database/src/lib/entities/delayed-job.entity.ts
@@ -1,17 +1,8 @@
-import {
-  AllowNull,
-  AutoIncrement,
-  Column,
-  Default,
-  Index,
-  Model,
-  PrimaryKey,
-  Table,
-} from 'sequelize-typescript';
-import { BIGINT, INTEGER, JSON, STRING, UUID } from 'sequelize';
+import { AllowNull, AutoIncrement, Column, Default, Index, Model, PrimaryKey, Table } from "sequelize-typescript";
+import { BIGINT, INTEGER, JSON, STRING, UUID } from "sequelize";
 
-@Table({ tableName: 'delayed_jobs', underscored: true })
-export class DelayedJob extends Model {
+@Table({ tableName: "delayed_jobs", underscored: true })
+export class DelayedJob extends Model<DelayedJob> {
   @PrimaryKey
   @AutoIncrement
   @Column(BIGINT.UNSIGNED)
@@ -21,7 +12,7 @@ export class DelayedJob extends Model {
   @Column(UUID)
   uuid: string;
 
-  @Default('pending')
+  @Default("pending")
   @Column(STRING)
   status: string;
 
diff --git a/libs/database/src/lib/entities/framework-user.entity.ts b/libs/database/src/lib/entities/framework-user.entity.ts
index ab6d5de..3d47000 100644
--- a/libs/database/src/lib/entities/framework-user.entity.ts
+++ b/libs/database/src/lib/entities/framework-user.entity.ts
@@ -1,10 +1,10 @@
-import { AutoIncrement, Column, ForeignKey, Model, PrimaryKey, Table } from 'sequelize-typescript';
-import { BIGINT } from 'sequelize';
-import { Framework } from './framework.entity';
-import { User } from './user.entity';
+import { AutoIncrement, Column, ForeignKey, Model, PrimaryKey, Table } from "sequelize-typescript";
+import { BIGINT } from "sequelize";
+import { Framework } from "./framework.entity";
+import { User } from "./user.entity";
 
-@Table({ tableName: 'framework_user', underscored: true })
-export class FrameworkUser extends Model {
+@Table({ tableName: "framework_user", underscored: true })
+export class FrameworkUser extends Model<FrameworkUser> {
   @PrimaryKey
   @AutoIncrement
   @Column(BIGINT.UNSIGNED)
diff --git a/libs/database/src/lib/entities/framework.entity.ts b/libs/database/src/lib/entities/framework.entity.ts
index 0c9f3fb..ad122ff 100644
--- a/libs/database/src/lib/entities/framework.entity.ts
+++ b/libs/database/src/lib/entities/framework.entity.ts
@@ -1,9 +1,9 @@
-import { AutoIncrement, Column, Model, PrimaryKey, Table } from 'sequelize-typescript';
-import { BIGINT, STRING } from 'sequelize';
+import { AutoIncrement, Column, Model, PrimaryKey, Table } from "sequelize-typescript";
+import { BIGINT, STRING } from "sequelize";
 
 // A quick stub to get the information needed for users/me
-@Table({ tableName: 'frameworks', underscored: true })
-export class Framework extends Model {
+@Table({ tableName: "frameworks", underscored: true })
+export class Framework extends Model<Framework> {
   @PrimaryKey
   @AutoIncrement
   @Column(BIGINT.UNSIGNED)
diff --git a/libs/database/src/lib/entities/indicator-output-field-monitoring.entity.ts b/libs/database/src/lib/entities/indicator-output-field-monitoring.entity.ts
index 4f8f5ea..eab96f0 100644
--- a/libs/database/src/lib/entities/indicator-output-field-monitoring.entity.ts
+++ b/libs/database/src/lib/entities/indicator-output-field-monitoring.entity.ts
@@ -4,7 +4,7 @@ import { SitePolygon } from "./site-polygon.entity";
 import { INDICATOR_SLUGS, IndicatorSlug } from "../constants";
 
 @Table({ tableName: "indicator_output_field_monitoring", underscored: true, paranoid: true })
-export class IndicatorOutputFieldMonitoring extends Model {
+export class IndicatorOutputFieldMonitoring extends Model<IndicatorOutputFieldMonitoring> {
   @PrimaryKey
   @AutoIncrement
   @Column(BIGINT.UNSIGNED)
diff --git a/libs/database/src/lib/entities/indicator-output-hectares.entity.ts b/libs/database/src/lib/entities/indicator-output-hectares.entity.ts
index 076c2b0..c463fe5 100644
--- a/libs/database/src/lib/entities/indicator-output-hectares.entity.ts
+++ b/libs/database/src/lib/entities/indicator-output-hectares.entity.ts
@@ -4,7 +4,7 @@ import { SitePolygon } from "./site-polygon.entity";
 import { INDICATOR_SLUGS, IndicatorSlug } from "../constants";
 
 @Table({ tableName: "indicator_output_hectares", underscored: true, paranoid: true })
-export class IndicatorOutputHectares extends Model {
+export class IndicatorOutputHectares extends Model<IndicatorOutputHectares> {
   @PrimaryKey
   @AutoIncrement
   @Column(BIGINT.UNSIGNED)
diff --git a/libs/database/src/lib/entities/indicator-output-msu-carbon.entity.ts b/libs/database/src/lib/entities/indicator-output-msu-carbon.entity.ts
index 052bc0d..8e7c001 100644
--- a/libs/database/src/lib/entities/indicator-output-msu-carbon.entity.ts
+++ b/libs/database/src/lib/entities/indicator-output-msu-carbon.entity.ts
@@ -4,7 +4,7 @@ import { SitePolygon } from "./site-polygon.entity";
 import { INDICATOR_SLUGS, IndicatorSlug } from "../constants";
 
 @Table({ tableName: "indicator_output_msu_carbon", underscored: true, paranoid: true })
-export class IndicatorOutputMsuCarbon extends Model {
+export class IndicatorOutputMsuCarbon extends Model<IndicatorOutputMsuCarbon> {
   @PrimaryKey
   @AutoIncrement
   @Column(BIGINT.UNSIGNED)
diff --git a/libs/database/src/lib/entities/indicator-output-tree-count.entity.ts b/libs/database/src/lib/entities/indicator-output-tree-count.entity.ts
index a3e308e..ed465b0 100644
--- a/libs/database/src/lib/entities/indicator-output-tree-count.entity.ts
+++ b/libs/database/src/lib/entities/indicator-output-tree-count.entity.ts
@@ -4,7 +4,7 @@ import { SitePolygon } from "./site-polygon.entity";
 import { INDICATOR_SLUGS, IndicatorSlug } from "../constants";
 
 @Table({ tableName: "indicator_output_tree_count", underscored: true, paranoid: true })
-export class IndicatorOutputTreeCount extends Model {
+export class IndicatorOutputTreeCount extends Model<IndicatorOutputTreeCount> {
   @PrimaryKey
   @AutoIncrement
   @Column(BIGINT.UNSIGNED)
diff --git a/libs/database/src/lib/entities/indicator-output-tree-cover-loss.entity.ts b/libs/database/src/lib/entities/indicator-output-tree-cover-loss.entity.ts
index 123abec..24d5012 100644
--- a/libs/database/src/lib/entities/indicator-output-tree-cover-loss.entity.ts
+++ b/libs/database/src/lib/entities/indicator-output-tree-cover-loss.entity.ts
@@ -4,7 +4,7 @@ import { SitePolygon } from "./site-polygon.entity";
 import { INDICATOR_SLUGS, IndicatorSlug } from "../constants";
 
 @Table({ tableName: "indicator_output_tree_cover_loss", underscored: true, paranoid: true })
-export class IndicatorOutputTreeCoverLoss extends Model {
+export class IndicatorOutputTreeCoverLoss extends Model<IndicatorOutputTreeCoverLoss> {
   @PrimaryKey
   @AutoIncrement
   @Column(BIGINT.UNSIGNED)
diff --git a/libs/database/src/lib/entities/indicator-output-tree-cover.entity.ts b/libs/database/src/lib/entities/indicator-output-tree-cover.entity.ts
index 12c867f..508eb6e 100644
--- a/libs/database/src/lib/entities/indicator-output-tree-cover.entity.ts
+++ b/libs/database/src/lib/entities/indicator-output-tree-cover.entity.ts
@@ -4,7 +4,7 @@ import { SitePolygon } from "./site-polygon.entity";
 import { INDICATOR_SLUGS, IndicatorSlug } from "../constants";
 
 @Table({ tableName: "indicator_output_tree_cover", underscored: true, paranoid: true })
-export class IndicatorOutputTreeCover extends Model {
+export class IndicatorOutputTreeCover extends Model<IndicatorOutputTreeCover> {
   @PrimaryKey
   @AutoIncrement
   @Column(BIGINT.UNSIGNED)
diff --git a/libs/database/src/lib/entities/model-has-role.entity.ts b/libs/database/src/lib/entities/model-has-role.entity.ts
index a537649..519b291 100644
--- a/libs/database/src/lib/entities/model-has-role.entity.ts
+++ b/libs/database/src/lib/entities/model-has-role.entity.ts
@@ -1,9 +1,9 @@
-import { Column, ForeignKey, Model, Table } from 'sequelize-typescript';
-import { Role } from './role.entity';
-import { BIGINT, STRING } from 'sequelize';
+import { Column, ForeignKey, Model, Table } from "sequelize-typescript";
+import { Role } from "./role.entity";
+import { BIGINT, STRING } from "sequelize";
 
-@Table({ tableName: 'model_has_roles', underscored: true, timestamps: false })
-export class ModelHasRole extends Model {
+@Table({ tableName: "model_has_roles", underscored: true, timestamps: false })
+export class ModelHasRole extends Model<ModelHasRole> {
   @ForeignKey(() => Role)
   @Column({ type: BIGINT.UNSIGNED, primaryKey: true })
   roleId: number;
diff --git a/libs/database/src/lib/entities/organisation-user.entity.ts b/libs/database/src/lib/entities/organisation-user.entity.ts
index 2a4961e..77ee105 100644
--- a/libs/database/src/lib/entities/organisation-user.entity.ts
+++ b/libs/database/src/lib/entities/organisation-user.entity.ts
@@ -1,10 +1,10 @@
-import { AutoIncrement, Column, ForeignKey, Model, PrimaryKey, Table } from 'sequelize-typescript';
-import { BIGINT, STRING } from 'sequelize';
-import { Organisation } from './organisation.entity';
-import { User } from './user.entity';
+import { AutoIncrement, Column, ForeignKey, Model, PrimaryKey, Table } from "sequelize-typescript";
+import { BIGINT, STRING } from "sequelize";
+import { Organisation } from "./organisation.entity";
+import { User } from "./user.entity";
 
-@Table({ tableName: 'organisation_user', underscored: true, timestamps: false })
-export class OrganisationUser extends Model {
+@Table({ tableName: "organisation_user", underscored: true, timestamps: false })
+export class OrganisationUser extends Model<OrganisationUser> {
   @PrimaryKey
   @AutoIncrement
   @Column(BIGINT.UNSIGNED)
diff --git a/libs/database/src/lib/entities/organisation.entity.ts b/libs/database/src/lib/entities/organisation.entity.ts
index f22f39d..4cfb401 100644
--- a/libs/database/src/lib/entities/organisation.entity.ts
+++ b/libs/database/src/lib/entities/organisation.entity.ts
@@ -2,7 +2,7 @@ import { AllowNull, AutoIncrement, Column, Default, Index, Model, PrimaryKey, Ta
 import { BIGINT, BOOLEAN, DATE, DECIMAL, ENUM, INTEGER, STRING, TEXT, TINYINT, UUID } from "sequelize";
 
 @Table({ tableName: "organisations", underscored: true })
-export class Organisation extends Model {
+export class Organisation extends Model<Organisation> {
   @PrimaryKey
   @AutoIncrement
   @Column(BIGINT.UNSIGNED)
diff --git a/libs/database/src/lib/entities/permission.entity.ts b/libs/database/src/lib/entities/permission.entity.ts
index f7f4383..8694fa6 100644
--- a/libs/database/src/lib/entities/permission.entity.ts
+++ b/libs/database/src/lib/entities/permission.entity.ts
@@ -1,8 +1,8 @@
-import { AutoIncrement, Column, Model, PrimaryKey, Table } from 'sequelize-typescript';
-import { BIGINT, QueryTypes, STRING } from 'sequelize';
+import { AutoIncrement, Column, Model, PrimaryKey, Table } from "sequelize-typescript";
+import { BIGINT, QueryTypes, STRING } from "sequelize";
 
-@Table({ tableName: 'permissions', underscored: true })
-export class Permission extends Model {
+@Table({ tableName: "permissions", underscored: true })
+export class Permission extends Model<Permission> {
   @PrimaryKey
   @AutoIncrement
   @Column(BIGINT.UNSIGNED)
@@ -35,8 +35,8 @@ export class Permission extends Model {
             model_has_roles.model_id = :modelId
     `,
       {
-        replacements: { modelType: 'App\\Models\\V2\\User', modelId: userId },
-        type: QueryTypes.SELECT,
+        replacements: { modelType: "App\\Models\\V2\\User", modelId: userId },
+        type: QueryTypes.SELECT
       }
     )) as { name: string }[];
 
diff --git a/libs/database/src/lib/entities/point-geometry.entity.ts b/libs/database/src/lib/entities/point-geometry.entity.ts
index 6d37d3d..16822fe 100644
--- a/libs/database/src/lib/entities/point-geometry.entity.ts
+++ b/libs/database/src/lib/entities/point-geometry.entity.ts
@@ -4,7 +4,7 @@ import { Point } from "geojson";
 import { User } from "./user.entity";
 
 @Table({ tableName: "point_geometry", underscored: true, paranoid: true })
-export class PointGeometry extends Model {
+export class PointGeometry extends Model<PointGeometry> {
   @PrimaryKey
   @AutoIncrement
   @Column(BIGINT.UNSIGNED)
diff --git a/libs/database/src/lib/entities/polygon-geometry.entity.ts b/libs/database/src/lib/entities/polygon-geometry.entity.ts
index 27b3ea4..719ece0 100644
--- a/libs/database/src/lib/entities/polygon-geometry.entity.ts
+++ b/libs/database/src/lib/entities/polygon-geometry.entity.ts
@@ -4,7 +4,7 @@ import { Polygon } from "geojson";
 import { User } from "./user.entity";
 
 @Table({ tableName: "polygon_geometry", underscored: true, paranoid: true })
-export class PolygonGeometry extends Model {
+export class PolygonGeometry extends Model<PolygonGeometry> {
   @PrimaryKey
   @AutoIncrement
   @Column(BIGINT.UNSIGNED)
diff --git a/libs/database/src/lib/entities/project-user.entity.ts b/libs/database/src/lib/entities/project-user.entity.ts
index c2626a7..3f49d8d 100644
--- a/libs/database/src/lib/entities/project-user.entity.ts
+++ b/libs/database/src/lib/entities/project-user.entity.ts
@@ -1,18 +1,10 @@
-import {
-  AllowNull,
-  AutoIncrement,
-  Column, Default,
-  ForeignKey,
-  Model,
-  PrimaryKey,
-  Table
-} from 'sequelize-typescript';
-import { Project } from './project.entity';
-import { User } from './user.entity';
-import { BIGINT, BOOLEAN, STRING } from 'sequelize';
+import { AllowNull, AutoIncrement, Column, Default, ForeignKey, Model, PrimaryKey, Table } from "sequelize-typescript";
+import { Project } from "./project.entity";
+import { User } from "./user.entity";
+import { BIGINT, BOOLEAN, STRING } from "sequelize";
 
-@Table({ tableName: 'v2_project_users', underscored: true })
-export class ProjectUser extends Model {
+@Table({ tableName: "v2_project_users", underscored: true })
+export class ProjectUser extends Model<ProjectUser> {
   @PrimaryKey
   @AutoIncrement
   @Column(BIGINT.UNSIGNED)
diff --git a/libs/database/src/lib/entities/project.entity.ts b/libs/database/src/lib/entities/project.entity.ts
index 05ec6e3..464a41f 100644
--- a/libs/database/src/lib/entities/project.entity.ts
+++ b/libs/database/src/lib/entities/project.entity.ts
@@ -3,7 +3,7 @@ import { BIGINT, BOOLEAN, STRING, UUID } from "sequelize";
 
 // A quick stub to get the information needed for users/me
 @Table({ tableName: "v2_projects", underscored: true, paranoid: true })
-export class Project extends Model {
+export class Project extends Model<Project> {
   @PrimaryKey
   @AutoIncrement
   @Column(BIGINT.UNSIGNED)
diff --git a/libs/database/src/lib/entities/role.entity.ts b/libs/database/src/lib/entities/role.entity.ts
index 0a93c93..c18db6f 100644
--- a/libs/database/src/lib/entities/role.entity.ts
+++ b/libs/database/src/lib/entities/role.entity.ts
@@ -1,14 +1,8 @@
-import {
-  AutoIncrement,
-  Column,
-  Model,
-  PrimaryKey,
-  Table,
-} from 'sequelize-typescript';
-import { BIGINT, STRING } from 'sequelize';
+import { AutoIncrement, Column, Model, PrimaryKey, Table } from "sequelize-typescript";
+import { BIGINT, STRING } from "sequelize";
 
-@Table({ tableName: 'roles', underscored: true })
-export class Role extends Model {
+@Table({ tableName: "roles", underscored: true })
+export class Role extends Model<Role> {
   @PrimaryKey
   @AutoIncrement
   @Column(BIGINT.UNSIGNED)
diff --git a/libs/database/src/lib/entities/site-polygon.entity.ts b/libs/database/src/lib/entities/site-polygon.entity.ts
index 9469237..5e4855c 100644
--- a/libs/database/src/lib/entities/site-polygon.entity.ts
+++ b/libs/database/src/lib/entities/site-polygon.entity.ts
@@ -33,7 +33,7 @@ export type Indicator =
   | IndicatorOutputMsuCarbon;
 
 @Table({ tableName: "site_polygon", underscored: true, paranoid: true })
-export class SitePolygon extends Model {
+export class SitePolygon extends Model<SitePolygon> {
   @PrimaryKey
   @AutoIncrement
   @Column(BIGINT.UNSIGNED)
diff --git a/libs/database/src/lib/entities/user.entity.ts b/libs/database/src/lib/entities/user.entity.ts
index ccc4858..282ea42 100644
--- a/libs/database/src/lib/entities/user.entity.ts
+++ b/libs/database/src/lib/entities/user.entity.ts
@@ -25,7 +25,7 @@ import { OrganisationUser } from "./organisation-user.entity";
 import { FrameworkUser } from "./framework-user.entity";
 
 @Table({ tableName: "users", underscored: true, paranoid: true })
-export class User extends Model {
+export class User extends Model<User> {
   @PrimaryKey
   @AutoIncrement
   @Column(BIGINT.UNSIGNED)
diff --git a/libs/database/tsconfig.json b/libs/database/tsconfig.json
index f5b8565..7678e97 100644
--- a/libs/database/tsconfig.json
+++ b/libs/database/tsconfig.json
@@ -7,7 +7,8 @@
     "noImplicitOverride": true,
     "noPropertyAccessFromIndexSignature": true,
     "noImplicitReturns": true,
-    "noFallthroughCasesInSwitch": true
+    "noFallthroughCasesInSwitch": true,
+    "strictPropertyInitialization": false
   },
   "files": [],
   "include": [],
diff --git a/setup-jest.ts b/setup-jest.ts
deleted file mode 100644
index 37f6084..0000000
--- a/setup-jest.ts
+++ /dev/null
@@ -1,26 +0,0 @@
-import { Sequelize } from 'sequelize-typescript';
-import { FactoryGirl, SequelizeAdapter } from 'factory-girl-ts';
-import * as Entities from '@terramatch-microservices/database/entities';
-
-let sequelize: Sequelize;
-
-beforeAll(async () => {
-  // To create this database, run the ./setup-test-database.sh script.
-  sequelize = new Sequelize({
-    dialect: 'mariadb',
-    host: 'localhost',
-    port: 3360,
-    username: 'wri',
-    password: 'wri',
-    database: 'terramatch_microservices_test',
-    models: Object.values(Entities),
-    logging: false,
-  })
-
-  await sequelize.sync();
-  FactoryGirl.setAdapter(new SequelizeAdapter());
-});
-
-afterAll(async () => {
-  await sequelize.close();
-});

From c581eb47c93d1ac95013a92e7fb90b8870460bef Mon Sep 17 00:00:00 2001
From: Nathan Curtis <nathan.curtis@gmail.com>
Date: Fri, 22 Nov 2024 13:26:23 -0800
Subject: [PATCH 14/16] [TM-1453] Implement bulk indicator upload.

---
 .../src/site-polygons/dto/indicators.dto.ts   |  68 ++++--
 .../dto/site-polygon-update.dto.ts            |  59 +++---
 .../site-polygon-query.builder.ts             | 161 +++++++++++++++
 .../site-polygons/site-polygons.controller.ts |  18 +-
 .../site-polygons/site-polygons.service.ts    | 193 ++++--------------
 5 files changed, 311 insertions(+), 188 deletions(-)
 create mode 100644 apps/research-service/src/site-polygons/site-polygon-query.builder.ts

diff --git a/apps/research-service/src/site-polygons/dto/indicators.dto.ts b/apps/research-service/src/site-polygons/dto/indicators.dto.ts
index f6be989..e161379 100644
--- a/apps/research-service/src/site-polygons/dto/indicators.dto.ts
+++ b/apps/research-service/src/site-polygons/dto/indicators.dto.ts
@@ -1,13 +1,16 @@
 import { ApiProperty } from "@nestjs/swagger";
 import { INDICATORS } from "@terramatch-microservices/database/constants";
+import { IsInt, IsNotEmpty, IsNumber, IsOptional, IsString } from "class-validator";
 
 export class IndicatorTreeCoverLossDto {
   @ApiProperty({ enum: [INDICATORS[2], INDICATORS[3]] })
   indicatorSlug: (typeof INDICATORS)[2] | (typeof INDICATORS)[3];
 
-  @ApiProperty({ example: "2024" })
+  @IsInt()
+  @ApiProperty({ example: 2024 })
   yearOfAnalysis: number;
 
+  @IsNotEmpty()
   @ApiProperty({
     type: "object",
     description: "Mapping of year of analysis to value.",
@@ -20,9 +23,11 @@ export class IndicatorHectaresDto {
   @ApiProperty({ enum: [INDICATORS[4], INDICATORS[5], INDICATORS[6]] })
   indicatorSlug: (typeof INDICATORS)[4] | (typeof INDICATORS)[5] | (typeof INDICATORS)[6];
 
+  @IsInt()
   @ApiProperty({ example: "2024" })
   yearOfAnalysis: number;
 
+  @IsNotEmpty()
   @ApiProperty({
     type: "object",
     description: "Mapping of area type (eco region, land use, etc) to hectares",
@@ -30,35 +35,51 @@ export class IndicatorHectaresDto {
   })
   value: Record<string, number>;
 }
-
 export class IndicatorTreeCountDto {
   @ApiProperty({ enum: [INDICATORS[7], INDICATORS[8]] })
   indicatorSlug: (typeof INDICATORS)[7] | (typeof INDICATORS)[8];
 
+  @IsInt()
   @ApiProperty({ example: "2024" })
   yearOfAnalysis: number;
 
+  @IsString()
+  @IsOptional()
   @ApiProperty()
   surveyType: string | null;
 
+  @IsNumber()
+  @IsOptional()
   @ApiProperty()
   surveyId: number | null;
 
+  @IsNumber()
+  @IsOptional()
   @ApiProperty()
   treeCount: number | null;
 
+  @IsString()
+  @IsOptional()
   @ApiProperty({ example: "types TBD" })
   uncertaintyType: string | null;
 
+  @IsString()
+  @IsOptional()
   @ApiProperty()
   imagerySource: string | null;
 
+  @IsString()
+  @IsOptional()
   @ApiProperty({ type: "url" })
   imageryId: string | null;
 
+  @IsString()
+  @IsOptional()
   @ApiProperty()
   projectPhase: string | null;
 
+  @IsNumber()
+  @IsOptional()
   @ApiProperty()
   confidence: number | null;
 }
@@ -67,15 +88,22 @@ export class IndicatorTreeCoverDto {
   @ApiProperty({ enum: [INDICATORS[1]] })
   indicatorSlug: (typeof INDICATORS)[1];
 
+  @IsInt()
   @ApiProperty({ example: "2024" })
   yearOfAnalysis: number;
 
+  @IsString()
+  @IsOptional()
   @ApiProperty({ example: "2024" })
   projectPhase: string | null;
 
+  @IsNumber()
+  @IsOptional()
   @ApiProperty()
   percentCover: number | null;
 
+  @IsNumber()
+  @IsOptional()
   @ApiProperty()
   plusMinusPercent: number | null;
 }
@@ -84,18 +112,27 @@ export class IndicatorFieldMonitoringDto {
   @ApiProperty({ enum: [INDICATORS[9]] })
   indicatorSlug: (typeof INDICATORS)[9];
 
+  @IsInt()
   @ApiProperty({ example: "2024" })
   yearOfAnalysis: number;
 
+  @IsNumber()
+  @IsOptional()
   @ApiProperty()
   treeCount: number | null;
 
+  @IsString()
+  @IsOptional()
   @ApiProperty()
   projectPhase: string | null;
 
+  @IsString()
+  @IsOptional()
   @ApiProperty()
   species: string | null;
 
+  @IsNumber()
+  @IsOptional()
   @ApiProperty()
   survivalRate: number | null;
 }
@@ -104,28 +141,35 @@ export class IndicatorMsuCarbonDto {
   @ApiProperty({ enum: [INDICATORS[10]] })
   indicatorSlug: (typeof INDICATORS)[10];
 
+  @IsInt()
   @ApiProperty({ example: "2024" })
   yearOfAnalysis: number;
 
+  @IsNumber()
+  @IsOptional()
   @ApiProperty()
   carbonOutput: number | null;
 
+  @IsString()
+  @IsOptional()
   @ApiProperty()
   projectPhase: string | null;
 
+  @IsNumber()
+  @IsOptional()
   @ApiProperty()
   confidence: number | null;
 }
 
 export const INDICATOR_DTOS = {
-  [INDICATORS[1]]: IndicatorTreeCoverDto.prototype,
-  [INDICATORS[2]]: IndicatorTreeCoverLossDto.prototype,
-  [INDICATORS[3]]: IndicatorTreeCoverLossDto.prototype,
-  [INDICATORS[4]]: IndicatorHectaresDto.prototype,
-  [INDICATORS[5]]: IndicatorHectaresDto.prototype,
-  [INDICATORS[6]]: IndicatorHectaresDto.prototype,
-  [INDICATORS[7]]: IndicatorTreeCountDto.prototype,
-  [INDICATORS[8]]: IndicatorTreeCountDto.prototype,
-  [INDICATORS[9]]: IndicatorFieldMonitoringDto.prototype,
-  [INDICATORS[10]]: IndicatorMsuCarbonDto.prototype
+  [INDICATORS[1]]: IndicatorTreeCoverDto,
+  [INDICATORS[2]]: IndicatorTreeCoverLossDto,
+  [INDICATORS[3]]: IndicatorTreeCoverLossDto,
+  [INDICATORS[4]]: IndicatorHectaresDto,
+  [INDICATORS[5]]: IndicatorHectaresDto,
+  [INDICATORS[6]]: IndicatorHectaresDto,
+  [INDICATORS[7]]: IndicatorTreeCountDto,
+  [INDICATORS[8]]: IndicatorTreeCountDto,
+  [INDICATORS[9]]: IndicatorFieldMonitoringDto,
+  [INDICATORS[10]]: IndicatorMsuCarbonDto
 };
diff --git a/apps/research-service/src/site-polygons/dto/site-polygon-update.dto.ts b/apps/research-service/src/site-polygons/dto/site-polygon-update.dto.ts
index 6e23a0d..2b2a0e2 100644
--- a/apps/research-service/src/site-polygons/dto/site-polygon-update.dto.ts
+++ b/apps/research-service/src/site-polygons/dto/site-polygon-update.dto.ts
@@ -1,48 +1,55 @@
-import { ApiProperty } from '@nestjs/swagger';
-import {
-  IndicatorFieldMonitoringDto,
-  IndicatorHectaresDto, IndicatorMsuCarbonDto,
-  IndicatorTreeCountDto, IndicatorTreeCoverDto,
-  IndicatorTreeCoverLossDto
-} from './indicators.dto';
+import { ApiProperty } from "@nestjs/swagger";
+import { IndicatorDto } from "./site-polygon.dto";
+import { Equals, IsArray, IsUUID, ValidateNested } from "class-validator";
+import { Type } from "class-transformer";
+import { INDICATOR_DTOS } from "./indicators.dto";
 
 class SitePolygonUpdateAttributes {
+  @IsArray()
+  @ValidateNested()
+  @Type(() => Object, {
+    keepDiscriminatorProperty: true,
+    discriminator: {
+      property: "indicatorSlug",
+      subTypes: Object.entries(INDICATOR_DTOS).map(([name, value]) => ({ name, value }))
+    }
+  })
   @ApiProperty({
-    type: 'array',
+    type: "array",
     items: {
       oneOf: [
-        { $ref: '#/components/schemas/IndicatorTreeCoverLossDto' },
-        { $ref: '#/components/schemas/IndicatorHectaresDto' },
-        { $ref: '#/components/schemas/IndicatorTreeCountDto' },
-        { $ref: '#/components/schemas/IndicatorTreeCoverDto' },
-        { $ref: '#/components/schemas/IndicatorFieldMonitoringDto' },
-        { $ref: '#/components/schemas/IndicatorMsuCarbonDto' },
+        { $ref: "#/components/schemas/IndicatorTreeCoverLossDto" },
+        { $ref: "#/components/schemas/IndicatorHectaresDto" },
+        { $ref: "#/components/schemas/IndicatorTreeCountDto" },
+        { $ref: "#/components/schemas/IndicatorTreeCoverDto" },
+        { $ref: "#/components/schemas/IndicatorFieldMonitoringDto" },
+        { $ref: "#/components/schemas/IndicatorMsuCarbonDto" }
       ]
     },
-    description: 'All indicators to update for this polygon'
+    description: "All indicators to update for this polygon"
   })
-  indicators: (
-    IndicatorTreeCoverLossDto |
-    IndicatorHectaresDto |
-    IndicatorTreeCountDto |
-    IndicatorTreeCoverDto |
-    IndicatorFieldMonitoringDto |
-    IndicatorMsuCarbonDto
-  )[];
+  indicators: IndicatorDto[];
 }
 
 class SitePolygonUpdate {
-  @ApiProperty({ enum: ['sitePolygons'] })
-  type: 'sitePolygons';
+  @Equals("sitePolygons")
+  @ApiProperty({ enum: ["sitePolygons"] })
+  type: string;
 
-  @ApiProperty({ format: 'uuid' })
+  @IsUUID()
+  @ApiProperty({ format: "uuid" })
   id: string;
 
+  @ValidateNested()
+  @Type(() => SitePolygonUpdateAttributes)
   @ApiProperty({ type: () => SitePolygonUpdateAttributes })
   attributes: SitePolygonUpdateAttributes;
 }
 
 export class SitePolygonBulkUpdateBodyDto {
+  @IsArray()
+  @ValidateNested()
+  @Type(() => SitePolygonUpdate)
   @ApiProperty({ isArray: true, type: () => SitePolygonUpdate })
   data: SitePolygonUpdate[];
 }
diff --git a/apps/research-service/src/site-polygons/site-polygon-query.builder.ts b/apps/research-service/src/site-polygons/site-polygon-query.builder.ts
new file mode 100644
index 0000000..bcf3fcf
--- /dev/null
+++ b/apps/research-service/src/site-polygons/site-polygon-query.builder.ts
@@ -0,0 +1,161 @@
+import { Attributes, Filterable, FindOptions, IncludeOptions, literal, Op, WhereOptions } from "sequelize";
+import {
+  IndicatorOutputFieldMonitoring,
+  IndicatorOutputHectares,
+  IndicatorOutputMsuCarbon,
+  IndicatorOutputTreeCount,
+  IndicatorOutputTreeCover,
+  IndicatorOutputTreeCoverLoss,
+  PolygonGeometry,
+  Project,
+  Site,
+  SitePolygon,
+  SiteReport,
+  TreeSpecies
+} from "@terramatch-microservices/database/entities";
+import { IndicatorSlug, PolygonStatus } from "@terramatch-microservices/database/constants";
+import { uniq } from "lodash";
+import { BadRequestException } from "@nestjs/common";
+
+type IndicatorModelClass =
+  | typeof IndicatorOutputTreeCover
+  | typeof IndicatorOutputTreeCoverLoss
+  | typeof IndicatorOutputHectares
+  | typeof IndicatorOutputTreeCount
+  | typeof IndicatorOutputFieldMonitoring
+  | typeof IndicatorOutputMsuCarbon;
+
+export const INDICATOR_MODEL_CLASSES: { [Slug in IndicatorSlug]: IndicatorModelClass } = {
+  treeCover: IndicatorOutputTreeCover,
+  treeCoverLoss: IndicatorOutputTreeCoverLoss,
+  treeCoverLossFires: IndicatorOutputTreeCoverLoss,
+  restorationByEcoRegion: IndicatorOutputHectares,
+  restorationByStrategy: IndicatorOutputHectares,
+  restorationByLandUse: IndicatorOutputHectares,
+  treeCount: IndicatorOutputTreeCount,
+  earlyTreeVerification: IndicatorOutputTreeCount,
+  fieldMonitoring: IndicatorOutputFieldMonitoring,
+  msuCarbon: IndicatorOutputMsuCarbon
+};
+
+const INDICATOR_EXCLUDE_COLUMNS = ["id", "sitePolygonId", "createdAt", "updatedAt", "deletedAt"];
+
+export class SitePolygonQueryBuilder {
+  private siteJoin: IncludeOptions = {
+    model: Site,
+    include: [
+      { model: TreeSpecies, attributes: ["name", "amount"] },
+      {
+        model: SiteReport,
+        include: [{ model: TreeSpecies, attributes: ["name", "amount"] }],
+        attributes: ["dueAt", "submittedAt"]
+      }
+    ],
+    attributes: ["projectId"],
+    required: true
+  };
+
+  private findOptions: FindOptions<Attributes<SitePolygon>> = {
+    include: [
+      { model: IndicatorOutputFieldMonitoring, attributes: { exclude: INDICATOR_EXCLUDE_COLUMNS } },
+      { model: IndicatorOutputHectares, attributes: { exclude: INDICATOR_EXCLUDE_COLUMNS } },
+      { model: IndicatorOutputMsuCarbon, attributes: { exclude: INDICATOR_EXCLUDE_COLUMNS } },
+      { model: IndicatorOutputTreeCount, attributes: { exclude: INDICATOR_EXCLUDE_COLUMNS } },
+      { model: IndicatorOutputTreeCover, attributes: { exclude: INDICATOR_EXCLUDE_COLUMNS } },
+      { model: IndicatorOutputTreeCoverLoss, attributes: { exclude: INDICATOR_EXCLUDE_COLUMNS } },
+      { model: PolygonGeometry, attributes: ["polygon"], required: true },
+      this.siteJoin
+    ]
+  };
+
+  constructor(pageSize: number) {
+    this.findOptions.limit = pageSize;
+  }
+
+  async excludeTestProjects() {
+    // avoid joining against the entire project table by doing a quick query first. The number of test projects is small
+    const testProjects = await Project.findAll({ where: { isTest: true }, attributes: ["id"] });
+    this.where({ projectId: { [Op.notIn]: testProjects.map(({ id }) => id) } }, this.siteJoin);
+    return this;
+  }
+
+  async filterProjectUuids(projectUuids: string[]) {
+    const filterProjects = await Project.findAll({
+      where: { uuid: { [Op.in]: projectUuids } },
+      attributes: ["id"]
+    });
+    this.where({ projectId: { [Op.in]: filterProjects.map(({ id }) => id) } }, this.siteJoin);
+    return this;
+  }
+
+  hasStatuses(polygonStatuses?: PolygonStatus[]) {
+    if (polygonStatuses != null) this.where({ status: { [Op.in]: polygonStatuses } });
+    return this;
+  }
+
+  modifiedSince(date?: Date) {
+    if (date != null) this.where({ updatedAt: { [Op.gte]: date } });
+    return this;
+  }
+
+  isMissingIndicators(indicatorSlugs?: IndicatorSlug[]) {
+    if (indicatorSlugs != null) {
+      const literals = uniq(indicatorSlugs).map(slug => {
+        const table = INDICATOR_MODEL_CLASSES[slug]?.tableName;
+        if (table == null) throw new BadRequestException(`Unrecognized indicator slug: ${slug}`);
+
+        return literal(
+          `(SELECT COUNT(*) = 0 from ${table} WHERE indicator_slug = "${slug}" AND site_polygon_id = SitePolygon.id)`
+        );
+      });
+      this.where({ [Op.and]: literals });
+    }
+    return this;
+  }
+
+  async touchesBoundary(polygonUuid?: string) {
+    if (polygonUuid != null) {
+      // This check isn't strictly necessary for constructing the query, but we do want to throw a useful
+      // error to the caller if the polygonUuid doesn't exist, and simply mixing it into the query won't
+      // do it
+      if ((await PolygonGeometry.count({ where: { uuid: polygonUuid } })) === 0) {
+        throw new BadRequestException(`Unrecognized polygon UUID: ${polygonUuid}`);
+      }
+
+      this.where({
+        [Op.and]: [
+          literal(
+            `(SELECT ST_INTERSECTS(polygon.geom, (SELECT geom FROM polygon_geometry WHERE uuid = "${polygonUuid}")))`
+          )
+        ]
+      });
+    }
+    return this;
+  }
+
+  async pageAfter(pageAfter: string) {
+    const sitePolygon = await SitePolygon.findOne({
+      where: { uuid: pageAfter },
+      attributes: ["id"]
+    });
+    if (sitePolygon == null) throw new BadRequestException("pageAfter polygon not found");
+    this.where({ id: { [Op.gt]: sitePolygon.id } });
+    return this;
+  }
+
+  async execute(): Promise<SitePolygon[]> {
+    return await SitePolygon.findAll(this.findOptions);
+  }
+
+  private where(options: WhereOptions, filterable: Filterable = this.findOptions) {
+    if (filterable.where == null) filterable.where = {};
+
+    const clauses = { ...options };
+    if (clauses[Op.and] != null && filterable.where[Op.and] != null) {
+      // For this builder, we only use arrays of literals with Op.and, so we can simply merge the arrays
+      clauses[Op.and] = [...filterable.where[Op.and], ...clauses[Op.and]];
+    }
+
+    Object.assign(filterable.where, clauses);
+  }
+}
diff --git a/apps/research-service/src/site-polygons/site-polygons.controller.ts b/apps/research-service/src/site-polygons/site-polygons.controller.ts
index 8e79651..f960b93 100644
--- a/apps/research-service/src/site-polygons/site-polygons.controller.ts
+++ b/apps/research-service/src/site-polygons/site-polygons.controller.ts
@@ -3,7 +3,7 @@ import {
   Body,
   Controller,
   Get,
-  NotImplementedException,
+  NotFoundException,
   Patch,
   Query,
   UnauthorizedException
@@ -96,7 +96,21 @@ export class SitePolygonsController {
   })
   @ApiOkResponse()
   @ApiException(() => UnauthorizedException, { description: "Authentication failed." })
+  @ApiException(() => BadRequestException, { description: "One or more of the data payload members has a problem." })
+  @ApiException(() => NotFoundException, { description: "A site polygon specified in the data was not found." })
   async bulkUpdate(@Body() updatePayload: SitePolygonBulkUpdateBodyDto): Promise<void> {
-    throw new NotImplementedException();
+    await this.policyService.authorize("updateAll", SitePolygon);
+
+    await this.sitePolygonService.transaction(async transaction => {
+      const updates: Promise<void>[] = [];
+      for (const update of updatePayload.data) {
+        for (const indicator of update.attributes.indicators) {
+          console.log("FOUND INDICATOR", indicator);
+          updates.push(this.sitePolygonService.updateIndicator(update.id, indicator, transaction));
+        }
+      }
+
+      await Promise.all(updates);
+    });
   }
 }
diff --git a/apps/research-service/src/site-polygons/site-polygons.service.ts b/apps/research-service/src/site-polygons/site-polygons.service.ts
index 954fb64..b97e287 100644
--- a/apps/research-service/src/site-polygons/site-polygons.service.ts
+++ b/apps/research-service/src/site-polygons/site-polygons.service.ts
@@ -1,152 +1,11 @@
-import { BadRequestException, Injectable, Type } from "@nestjs/common";
-import {
-  IndicatorOutputFieldMonitoring,
-  IndicatorOutputHectares,
-  IndicatorOutputMsuCarbon,
-  IndicatorOutputTreeCount,
-  IndicatorOutputTreeCover,
-  IndicatorOutputTreeCoverLoss,
-  PolygonGeometry,
-  Project,
-  Site,
-  SitePolygon,
-  SiteReport,
-  TreeSpecies
-} from "@terramatch-microservices/database/entities";
-import { Attributes, Filterable, FindOptions, IncludeOptions, literal, Op, WhereOptions } from "sequelize";
+import { BadRequestException, Injectable, NotFoundException, Type } from "@nestjs/common";
+import { SitePolygon } from "@terramatch-microservices/database/entities";
 import { IndicatorDto, ReportingPeriodDto, TreeSpeciesDto } from "./dto/site-polygon.dto";
 import { INDICATOR_DTOS } from "./dto/indicators.dto";
 import { ModelPropertiesAccessor } from "@nestjs/swagger/dist/services/model-properties-accessor";
-import { pick, uniq } from "lodash";
-import { IndicatorSlug, PolygonStatus } from "@terramatch-microservices/database/constants";
-
-const INDICATOR_TABLES: { [Slug in IndicatorSlug]: string } = {
-  treeCover: "indicator_output_tree_cover",
-  treeCoverLoss: "indicator_output_tree_cover_loss",
-  treeCoverLossFires: "indicator_output_tree_cover_loss",
-  restorationByEcoRegion: "indicator_output_hectares",
-  restorationByStrategy: "indicator_output_hectares",
-  restorationByLandUse: "indicator_output_hectares",
-  treeCount: "indicator_output_tree_count",
-  earlyTreeVerification: "indicator_output_tree_count",
-  fieldMonitoring: "indicator_output_field_monitoring",
-  msuCarbon: "indicator_output_msu_carbon"
-};
-
-const INDICATOR_EXCLUDE_COLUMNS = ["id", "sitePolygonId", "createdAt", "updatedAt", "deletedAt"];
-
-export class SitePolygonQueryBuilder {
-  private siteJoin: IncludeOptions = {
-    model: Site,
-    include: [
-      { model: TreeSpecies, attributes: ["name", "amount"] },
-      {
-        model: SiteReport,
-        include: [{ model: TreeSpecies, attributes: ["name", "amount"] }],
-        attributes: ["dueAt", "submittedAt"]
-      }
-    ],
-    attributes: ["projectId"],
-    required: true
-  };
-  private findOptions: FindOptions<Attributes<SitePolygon>> = {
-    include: [
-      { model: IndicatorOutputFieldMonitoring, attributes: { exclude: INDICATOR_EXCLUDE_COLUMNS } },
-      { model: IndicatorOutputHectares, attributes: { exclude: INDICATOR_EXCLUDE_COLUMNS } },
-      { model: IndicatorOutputMsuCarbon, attributes: { exclude: INDICATOR_EXCLUDE_COLUMNS } },
-      { model: IndicatorOutputTreeCount, attributes: { exclude: INDICATOR_EXCLUDE_COLUMNS } },
-      { model: IndicatorOutputTreeCover, attributes: { exclude: INDICATOR_EXCLUDE_COLUMNS } },
-      { model: IndicatorOutputTreeCoverLoss, attributes: { exclude: INDICATOR_EXCLUDE_COLUMNS } },
-      { model: PolygonGeometry, attributes: ["polygon"], required: true },
-      this.siteJoin
-    ]
-  };
-
-  constructor(pageSize: number) {
-    this.findOptions.limit = pageSize;
-  }
-
-  async excludeTestProjects() {
-    // avoid joining against the entire project table by doing a quick query first. The number of test projects is small
-    const testProjects = await Project.findAll({ where: { isTest: true }, attributes: ["id"] });
-    this.where({ projectId: { [Op.notIn]: testProjects.map(({ id }) => id) } }, this.siteJoin);
-    return this;
-  }
-
-  async filterProjectUuids(projectUuids: string[]) {
-    const filterProjects = await Project.findAll({ where: { uuid: { [Op.in]: projectUuids } }, attributes: ["id"] });
-    this.where({ projectId: { [Op.in]: filterProjects.map(({ id }) => id) } }, this.siteJoin);
-    return this;
-  }
-
-  hasStatuses(polygonStatuses?: PolygonStatus[]) {
-    if (polygonStatuses != null) this.where({ status: { [Op.in]: polygonStatuses } });
-    return this;
-  }
-
-  modifiedSince(date?: Date) {
-    if (date != null) this.where({ updatedAt: { [Op.gte]: date } });
-    return this;
-  }
-
-  isMissingIndicators(indicatorSlugs?: IndicatorSlug[]) {
-    if (indicatorSlugs != null) {
-      const literals = uniq(indicatorSlugs).map(slug => {
-        const table = INDICATOR_TABLES[slug];
-        if (table == null) throw new BadRequestException(`Unrecognized indicator slug: ${slug}`);
-
-        return literal(
-          `(SELECT COUNT(*) = 0 from ${table} WHERE indicator_slug = "${slug}" AND site_polygon_id = SitePolygon.id)`
-        );
-      });
-      this.where({ [Op.and]: literals });
-    }
-    return this;
-  }
-
-  async touchesBoundary(polygonUuid?: string) {
-    if (polygonUuid != null) {
-      // This check isn't strictly necessary for constructing the query, but we do want to throw a useful
-      // error to the caller if the polygonUuid doesn't exist, and simply mixing it into the query won't
-      // do it
-      if ((await PolygonGeometry.count({ where: { uuid: polygonUuid } })) === 0) {
-        throw new BadRequestException(`Unrecognized polygon UUID: ${polygonUuid}`);
-      }
-
-      this.where({
-        [Op.and]: [
-          literal(
-            `(SELECT ST_INTERSECTS(polygon.geom, (SELECT geom FROM polygon_geometry WHERE uuid = "${polygonUuid}")))`
-          )
-        ]
-      });
-    }
-    return this;
-  }
-
-  async pageAfter(pageAfter: string) {
-    const sitePolygon = await SitePolygon.findOne({ where: { uuid: pageAfter }, attributes: ["id"] });
-    if (sitePolygon == null) throw new BadRequestException("pageAfter polygon not found");
-    this.where({ id: { [Op.gt]: sitePolygon.id } });
-    return this;
-  }
-
-  async execute(): Promise<SitePolygon[]> {
-    return await SitePolygon.findAll(this.findOptions);
-  }
-
-  private where(options: WhereOptions, filterable: Filterable = this.findOptions) {
-    if (filterable.where == null) filterable.where = {};
-
-    const clauses = { ...options };
-    if (clauses[Op.and] != null && filterable.where[Op.and] != null) {
-      // For this builder, we only use arrays of literals with Op.and, so we can simply merge the arrays
-      clauses[Op.and] = [...filterable.where[Op.and], ...clauses[Op.and]];
-    }
-
-    Object.assign(filterable.where, clauses);
-  }
-}
+import { pick } from "lodash";
+import { INDICATOR_MODEL_CLASSES, SitePolygonQueryBuilder } from "./site-polygon-query.builder";
+import { Transaction } from "sequelize";
 
 @Injectable()
 export class SitePolygonsService {
@@ -161,8 +20,8 @@ export class SitePolygonsService {
     const indicators: IndicatorDto[] = [];
     for (const indicator of await sitePolygon.getIndicators()) {
       const DtoPrototype = INDICATOR_DTOS[indicator.indicatorSlug];
-      const fields = accessor.getModelProperties(DtoPrototype as unknown as Type<unknown>);
-      indicators.push(pick(indicator, fields) as typeof DtoPrototype);
+      const fields = accessor.getModelProperties(DtoPrototype.prototype as unknown as Type<unknown>);
+      indicators.push(pick(indicator, fields) as typeof DtoPrototype.prototype);
     }
 
     return indicators;
@@ -194,4 +53,42 @@ export class SitePolygonsService {
 
     return reportingPeriods;
   }
+
+  async updateIndicator(sitePolygonUuid: string, indicator: IndicatorDto, transaction?: Transaction): Promise<void> {
+    const accessor = new ModelPropertiesAccessor();
+    const { indicatorSlug, yearOfAnalysis } = indicator;
+    const { id: sitePolygonId } = (await SitePolygon.findOne({ where: { uuid: sitePolygonUuid } })) ?? {};
+    if (sitePolygonId == null) {
+      throw new NotFoundException(`SitePolygon not found for id: ${sitePolygonUuid}`);
+    }
+
+    const IndicatorClass = INDICATOR_MODEL_CLASSES[indicatorSlug];
+    if (IndicatorClass == null) {
+      throw new BadRequestException(`Model not found for indicator: ${indicator.indicatorSlug}`);
+    }
+
+    const model =
+      // @ts-expect-error The compiler is getting confused here; this is legal.
+      (await IndicatorClass.findOne({
+        where: { sitePolygonId, indicatorSlug, yearOfAnalysis }
+      })) ?? new IndicatorClass();
+    if (model.sitePolygonId == null) model.sitePolygonId = sitePolygonId;
+
+    const DtoPrototype = INDICATOR_DTOS[indicatorSlug];
+    const fields = accessor.getModelProperties(DtoPrototype as unknown as Type<unknown>);
+    Object.assign(model, pick(indicator, fields));
+    await model.save({ transaction });
+  }
+
+  async transaction<TReturn>(callback: (transaction: Transaction) => Promise<TReturn>) {
+    const transaction = await SitePolygon.sequelize.transaction();
+    try {
+      const result = await callback(transaction);
+      await transaction.commit();
+      return result;
+    } catch (e) {
+      await transaction.rollback();
+      throw e;
+    }
+  }
 }

From 2d055d1b8b290b66492959171419b566b2542e66 Mon Sep 17 00:00:00 2001
From: Nathan Curtis <nathan.curtis@gmail.com>
Date: Fri, 22 Nov 2024 14:45:08 -0800
Subject: [PATCH 15/16] [TM-1453] Tests for the bulk update endpoint and update
 indicator service method.

---
 .../site-polygons.controller.spec.ts          |  60 ++++++++++-
 .../site-polygons/site-polygons.controller.ts |   1 -
 .../site-polygons.service.spec.ts             | 101 +++++++++++++++++-
 .../site-polygons/site-polygons.service.ts    |   4 +-
 4 files changed, 159 insertions(+), 7 deletions(-)

diff --git a/apps/research-service/src/site-polygons/site-polygons.controller.spec.ts b/apps/research-service/src/site-polygons/site-polygons.controller.spec.ts
index 6d0005b..ac47aaa 100644
--- a/apps/research-service/src/site-polygons/site-polygons.controller.spec.ts
+++ b/apps/research-service/src/site-polygons/site-polygons.controller.spec.ts
@@ -3,10 +3,12 @@ import { SitePolygonsService } from "./site-polygons.service";
 import { createMock, DeepMocked } from "@golevelup/ts-jest";
 import { Test, TestingModule } from "@nestjs/testing";
 import { PolicyService } from "@terramatch-microservices/common";
-import { BadRequestException, NotImplementedException, UnauthorizedException } from "@nestjs/common";
+import { BadRequestException, UnauthorizedException } from "@nestjs/common";
 import { Resource } from "@terramatch-microservices/common/util";
 import { SitePolygon } from "@terramatch-microservices/database/entities";
 import { SitePolygonFactory } from "@terramatch-microservices/database/factories";
+import { SitePolygonBulkUpdateBodyDto } from "./dto/site-polygon-update.dto";
+import { Transaction } from "sequelize";
 
 describe("SitePolygonsController", () => {
   let controller: SitePolygonsController;
@@ -108,8 +110,60 @@ describe("SitePolygonsController", () => {
   });
 
   describe("bulkUpdate", () => {
-    it("Should throw", async () => {
-      await expect(controller.bulkUpdate(null)).rejects.toThrow(NotImplementedException);
+    it("Should authorize", async () => {
+      policyService.authorize.mockRejectedValue(new UnauthorizedException());
+      await expect(controller.bulkUpdate(null)).rejects.toThrow(UnauthorizedException);
+    });
+
+    it("should use a transaction for updates", async () => {
+      const transaction = {} as Transaction;
+      sitePolygonService.updateIndicator.mockResolvedValue();
+      sitePolygonService.transaction.mockImplementation(callback => callback(transaction));
+      const indicator = {
+        indicatorSlug: "restorationByLandUse",
+        yearOfAnalysis: 2025,
+        value: {
+          "Northern Acacia-Commiphora bushlands and thickets": 0.114
+        }
+      };
+      const payload = {
+        data: [{ type: "sitePolygons", id: "1234", attributes: { indicators: [indicator] } }]
+      } as SitePolygonBulkUpdateBodyDto;
+      await controller.bulkUpdate(payload);
+      expect(sitePolygonService.updateIndicator).toHaveBeenCalledWith("1234", indicator, transaction);
+    });
+
+    it("should call update for each indicator in the payload", async () => {
+      const transaction = {} as Transaction;
+      sitePolygonService.updateIndicator.mockResolvedValue();
+      sitePolygonService.transaction.mockImplementation(callback => callback(transaction));
+      const indicator1 = {
+        indicatorSlug: "restorationByLandUse",
+        yearOfAnalysis: 2025,
+        value: {
+          "Northern Acacia-Commiphora bushlands and thickets": 0.114
+        }
+      };
+      const indicator2 = {
+        indicatorSlug: "treeCoverLoss",
+        yearOfAnalysis: 2025,
+        value: {
+          "2023": 0.45,
+          "2024": 0.6,
+          "2025": 0.8
+        }
+      };
+      const payload = {
+        data: [
+          { type: "sitePolygons", id: "1234", attributes: { indicators: [indicator1, indicator2] } },
+          { type: "sitePolygons", id: "2345", attributes: { indicators: [indicator2] } }
+        ]
+      } as SitePolygonBulkUpdateBodyDto;
+      await controller.bulkUpdate(payload);
+      expect(sitePolygonService.updateIndicator).toHaveBeenCalledTimes(3);
+      expect(sitePolygonService.updateIndicator).toHaveBeenNthCalledWith(1, "1234", indicator1, transaction);
+      expect(sitePolygonService.updateIndicator).toHaveBeenNthCalledWith(2, "1234", indicator2, transaction);
+      expect(sitePolygonService.updateIndicator).toHaveBeenNthCalledWith(3, "2345", indicator2, transaction);
     });
   });
 });
diff --git a/apps/research-service/src/site-polygons/site-polygons.controller.ts b/apps/research-service/src/site-polygons/site-polygons.controller.ts
index f960b93..a5e6138 100644
--- a/apps/research-service/src/site-polygons/site-polygons.controller.ts
+++ b/apps/research-service/src/site-polygons/site-polygons.controller.ts
@@ -105,7 +105,6 @@ export class SitePolygonsController {
       const updates: Promise<void>[] = [];
       for (const update of updatePayload.data) {
         for (const indicator of update.attributes.indicators) {
-          console.log("FOUND INDICATOR", indicator);
           updates.push(this.sitePolygonService.updateIndicator(update.id, indicator, transaction));
         }
       }
diff --git a/apps/research-service/src/site-polygons/site-polygons.service.spec.ts b/apps/research-service/src/site-polygons/site-polygons.service.spec.ts
index 7721ec2..a000ea3 100644
--- a/apps/research-service/src/site-polygons/site-polygons.service.spec.ts
+++ b/apps/research-service/src/site-polygons/site-polygons.service.spec.ts
@@ -17,10 +17,11 @@ import {
   TreeSpeciesFactory
 } from "@terramatch-microservices/database/factories";
 import { Indicator, PolygonGeometry, SitePolygon, TreeSpecies } from "@terramatch-microservices/database/entities";
-import { BadRequestException } from "@nestjs/common";
+import { BadRequestException, NotFoundException } from "@nestjs/common";
 import { faker } from "@faker-js/faker";
 import { DateTime } from "luxon";
 import { IndicatorSlug } from "@terramatch-microservices/database/constants";
+import { IndicatorHectaresDto, IndicatorTreeCountDto, IndicatorTreeCoverLossDto } from "./dto/indicators.dto";
 
 describe("SitePolygonsService", () => {
   let service: SitePolygonsService;
@@ -329,4 +330,102 @@ describe("SitePolygonsService", () => {
     expect(result.length).toBe(1);
     expect(result[0].id).toBe(draftPoly2.id);
   });
+
+  it("should commit a transaction", async () => {
+    const commit = jest.fn();
+    // @ts-expect-error incomplete mock.
+    jest.spyOn(SitePolygon.sequelize, "transaction").mockResolvedValue({ commit });
+
+    const result = await service.transaction(async () => "result");
+    expect(result).toBe("result");
+    expect(commit).toHaveBeenCalled();
+  });
+
+  it("should roll back a transaction", async () => {
+    const rollback = jest.fn();
+    // @ts-expect-error incomplete mock
+    jest.spyOn(SitePolygon.sequelize, "transaction").mockResolvedValue({ rollback });
+
+    await expect(
+      service.transaction(async () => {
+        throw new Error("Test Exception");
+      })
+    ).rejects.toThrow("Test Exception");
+    expect(rollback).toHaveBeenCalled();
+  });
+
+  it("should throw if the site polygon is not found", async () => {
+    await expect(service.updateIndicator("asdfasdf", null)).rejects.toThrow(NotFoundException);
+  });
+
+  it("should throw if the indicator slug is invalid", async () => {
+    const { uuid } = await SitePolygonFactory.create();
+    // @ts-expect-error incomplete DTO object
+    await expect(service.updateIndicator(uuid, { indicatorSlug: "foobar" as IndicatorSlug })).rejects.toThrow(
+      BadRequestException
+    );
+  });
+
+  it("should create a new indicator row if none exists", async () => {
+    const sitePolygon = await SitePolygonFactory.create();
+    const dto = {
+      indicatorSlug: "treeCoverLoss",
+      yearOfAnalysis: 2025,
+      value: {
+        "2023": 0.45,
+        "2024": 0.6,
+        "2025": 0.8
+      }
+    } as IndicatorTreeCoverLossDto;
+    await service.updateIndicator(sitePolygon.uuid, dto);
+    const treeCoverLoss = await sitePolygon.$get("indicatorsTreeCoverLoss");
+    expect(treeCoverLoss.length).toBe(1);
+    expect(treeCoverLoss[0]).toMatchObject(dto);
+  });
+
+  it("should create a new indicator row if the yearOfAnalysis does not match", async () => {
+    const sitePolygon = await SitePolygonFactory.create();
+    const dto = {
+      indicatorSlug: "restorationByLandUse",
+      yearOfAnalysis: 2025,
+      value: {
+        "Northern Acacia-Commiphora bushlands and thickets": 0.114
+      }
+    } as IndicatorHectaresDto;
+    await IndicatorOutputHectaresFactory.create({
+      ...dto,
+      yearOfAnalysis: 2024,
+      sitePolygonId: sitePolygon.id
+    });
+    await service.updateIndicator(sitePolygon.uuid, dto);
+    const hectares = await sitePolygon.$get("indicatorsHectares");
+    expect(hectares.length).toBe(2);
+    expect(hectares[0]).toMatchObject({ ...dto, yearOfAnalysis: 2024 });
+    expect(hectares[1]).toMatchObject(dto);
+  });
+
+  it("should update an indicator if it already exists", async () => {
+    const sitePolygon = await SitePolygonFactory.create();
+    const dto = {
+      indicatorSlug: "treeCount",
+      yearOfAnalysis: 2024,
+      surveyType: "string",
+      surveyId: 1000,
+      treeCount: 5432,
+      uncertaintyType: "types TBD",
+      imagerySource: "maxar",
+      imageryId: "https://foo.bar/image",
+      projectPhase: "establishment",
+      confidence: 70
+    } as IndicatorTreeCountDto;
+    await IndicatorOutputTreeCountFactory.create({
+      ...dto,
+      sitePolygonId: sitePolygon.id,
+      confidence: 20
+    });
+    await service.updateIndicator(sitePolygon.uuid, dto);
+    const treeCount = await sitePolygon.$get("indicatorsTreeCount");
+    expect(treeCount.length).toBe(1);
+    expect(treeCount[0]).toMatchObject(dto);
+  });
 });
diff --git a/apps/research-service/src/site-polygons/site-polygons.service.ts b/apps/research-service/src/site-polygons/site-polygons.service.ts
index b97e287..357cba2 100644
--- a/apps/research-service/src/site-polygons/site-polygons.service.ts
+++ b/apps/research-service/src/site-polygons/site-polygons.service.ts
@@ -56,12 +56,12 @@ export class SitePolygonsService {
 
   async updateIndicator(sitePolygonUuid: string, indicator: IndicatorDto, transaction?: Transaction): Promise<void> {
     const accessor = new ModelPropertiesAccessor();
-    const { indicatorSlug, yearOfAnalysis } = indicator;
     const { id: sitePolygonId } = (await SitePolygon.findOne({ where: { uuid: sitePolygonUuid } })) ?? {};
     if (sitePolygonId == null) {
       throw new NotFoundException(`SitePolygon not found for id: ${sitePolygonUuid}`);
     }
 
+    const { indicatorSlug, yearOfAnalysis } = indicator;
     const IndicatorClass = INDICATOR_MODEL_CLASSES[indicatorSlug];
     if (IndicatorClass == null) {
       throw new BadRequestException(`Model not found for indicator: ${indicator.indicatorSlug}`);
@@ -75,7 +75,7 @@ export class SitePolygonsService {
     if (model.sitePolygonId == null) model.sitePolygonId = sitePolygonId;
 
     const DtoPrototype = INDICATOR_DTOS[indicatorSlug];
-    const fields = accessor.getModelProperties(DtoPrototype as unknown as Type<unknown>);
+    const fields = accessor.getModelProperties(DtoPrototype.prototype as unknown as Type<unknown>);
     Object.assign(model, pick(indicator, fields));
     await model.save({ transaction });
   }

From 2fa9abca3e72da9fd4f9e17afa4063c3e8adcd04 Mon Sep 17 00:00:00 2001
From: Nathan Curtis <nathan.curtis@gmail.com>
Date: Fri, 22 Nov 2024 14:52:06 -0800
Subject: [PATCH 16/16] [TM-1453] The database test needs to run without cache
 so it does its job of setting up the DB.

---
 .github/workflows/pull-request.yml                              | 2 +-
 .../research-service/src/site-polygons/site-polygons.service.ts | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml
index 425ea18..e968eb1 100644
--- a/.github/workflows/pull-request.yml
+++ b/.github/workflows/pull-request.yml
@@ -46,7 +46,7 @@ jobs:
       # in a clean way. For some reason, the `run-many` is necessary here. If this line simply uses
       # nx test database, the connection to the DB gets cut off before the sync is complete.
       - name: Sync DB Schema
-        run: NX_CLOUD_DISTRIBUTED_EXECUTION=false npx nx test database
+        run: NX_CLOUD_DISTRIBUTED_EXECUTION=false npx nx test database --no-cache
 
       - name: Test all
         run: NX_CLOUD_DISTRIBUTED_EXECUTION=false npx nx run-many -t test --coverage --passWithNoTests
diff --git a/apps/research-service/src/site-polygons/site-polygons.service.ts b/apps/research-service/src/site-polygons/site-polygons.service.ts
index 357cba2..6a50f95 100644
--- a/apps/research-service/src/site-polygons/site-polygons.service.ts
+++ b/apps/research-service/src/site-polygons/site-polygons.service.ts
@@ -64,7 +64,7 @@ export class SitePolygonsService {
     const { indicatorSlug, yearOfAnalysis } = indicator;
     const IndicatorClass = INDICATOR_MODEL_CLASSES[indicatorSlug];
     if (IndicatorClass == null) {
-      throw new BadRequestException(`Model not found for indicator: ${indicator.indicatorSlug}`);
+      throw new BadRequestException(`Model not found for indicator: ${indicatorSlug}`);
     }
 
     const model =