Merge pull request #321 from PhilanthropyDataCommons/sqlfluff

Lint SQL with SQLFluff

.github/workflows/lint.yml

@@ -24,6 +24,17 @@ jobs:
           node-version: 20.17.0
       - run: npm ci
       - run: npm run lint:prettier
+  sqlfluff:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+      - name: Install SQLFluff
+        run: 'pip install -r requirements.txt'
+      - name: Lint SQL
+        run: sqlfluff lint --format github-annotation-native src/database/
   swagger:
     runs-on: ubuntu-latest
     steps:

.sqlfluff

@@ -0,0 +1,22 @@
+[sqlfluff]
+dialect = postgres
+templater = placeholder
+exclude_rules =
+  layout.indent,
+  # allow use of keywords as identifiers when unambiguous
+  references.keywords,
+
+[sqlfluff:indentation]
+indent_unit = space
+tab_space_size = 2
+
+[sqlfluff:rules:convention.terminator]
+multiline_newline = False
+require_final_semicolon = True
+
+[sqlfluff:templater:placeholder]
+param_style = colon
+
+# placeholder values
+limit = 20
+offset = 0

.sqlfluffignore

@@ -0,0 +1,49 @@
+dist/
+node_modules/
+
+# Migrations are immutable, so already-committed migrations cannot be corrected.
+# https://github.com/zakpatterson/postgres-schema-migrations/blob/75f22a65d3a8ddd16356e74538316fec2ca0202e/README.md#hash-checks-for-previous-migrations
+src/database/migrations/0001-create-canonical_fields.sql
+src/database/migrations/0002-create-opportunities.sql
+src/database/migrations/0003-create-applicants.sql
+src/database/migrations/0004-create-application_forms.sql
+src/database/migrations/0005-create-application_form_fields.sql
+src/database/migrations/0006-create-proposals.sql
+src/database/migrations/0007-create-proposal_versions.sql
+src/database/migrations/0008-create-proposal_field_values.sql
+src/database/migrations/0009-alter-proposal_field_values-position.sql
+src/database/migrations/0010-alter-proposal_field_values-value_search.sql
+src/database/migrations/0011-rename-canonical_fields.sql
+src/database/migrations/0012-create-platform_provider_responses.sql
+src/database/migrations/0013-create-bulk_uploads.sql
+src/database/migrations/0014-alter-base_fields-description.sql
+src/database/migrations/0015-alter-bulk_uploads-source_key.sql
+src/database/migrations/0016-alter-bulk_uploads-file_size.sql
+src/database/migrations/0017-create-organizations.sql
+src/database/migrations/0018-remove-applicants.sql
+src/database/migrations/0019-create-organizations_proposals.sql
+src/database/migrations/0020-alter-base_fields.sql
+src/database/migrations/0021-alter-proposal_field_values.sql
+src/database/migrations/0022-create-users.sql
+src/database/migrations/0023-insert-system-user.sql
+src/database/migrations/0024-alter-bulk_uploads.sql
+src/database/migrations/0025-alter-proposals.sql
+src/database/migrations/0026-alter-base_fields-add-field_scope.sql
+src/database/migrations/0027-alter-organizations-rename-ein.sql
+src/database/migrations/0028-alter-field_type-enum.sql
+src/database/migrations/0029-create-base_field_localizations.sql
+src/database/migrations/0030-create-funders.sql
+src/database/migrations/0031-create-data_providers.sql
+src/database/migrations/0032-create-sources.sql
+src/database/migrations/0033-alter-proposal_versions-add-sourceId.sql
+src/database/migrations/0034-alter-bulk_uploads-add-source_id.sql
+src/database/migrations/0035-alter-proposal_versions-add-created_by.sql
+src/database/migrations/0036-create-function-drop_function.sql
+src/database/migrations/0037-alter-users-change-authentication_id-to-keycloak_user_id.sql
+src/database/migrations/0038-alter-users-use-keycloak_user_id-as-primary-key.sql
+src/database/migrations/0039-rename-organizations-to-changemakers.sql
+src/database/migrations/0040-create-permission-tables.sql
+src/database/migrations/0041-rename-bulk_uploads-to-bulk_upload_tasks.sql
+
+# This file is very large, and is intended to be removed soon.
+src/database/seeds/0001-insert-base_fields.sql

README.md

@@ -182,6 +182,10 @@ For example: `0001-create-users` or `0001-modify-users`
 
 In `/src/databases/seeds` there is seed or starter data. The contents can be run manually to help developers get data in their databases. The scripts are not referenced by the software and are included for convenience. The migrations must run prior to using seed scripts.
 
+We use [SQLFluff](https://docs.sqlfluff.com/en/stable/index.html) to lint our SQL files. See [Getting Started with SQLFluff](https://docs.sqlfluff.com/en/stable/gettingstarted.html) for instructions on setting it up.
+
+Once you have SQLFluff installed, you can run it with `npm run lint:sqlfluff`, and use `npm run format:sqlfluff` to automatically fix any linter errors it can.
+
 #### Node version
 
 We aim to use the "Active LTS" version of node. An exact version of node is specified in automated workflows and Dockerfile while a major version is specified in the .node-version. You should be able to use any minor version within the Active LTS version and might be able to use other major versions.

package.json

@@ -7,12 +7,14 @@
 		"build:node": "tsc -p tsconfig.json",
 		"build:assets": "copyfiles -u 1 \"src/**/*.sql\" \"src/**/.keep\" \"src/public/**\" dist",
 		"cleanup": "rimraf dist",
-		"format": "prettier . --write",
+		"format:prettier": "prettier . --write",
+		"format:sqlfluff": "sqlfluff fix src/database/",
 		"lint": "npm run lint:eslint && npm run lint:prettier && npm run lint:tsc && npm run lint:swagger",
 		"lint:eslint": "eslint ./src --ext .ts --max-warnings=0",
 		"lint:swagger": "swagger-spec-validator src/openapi.json",
 		"lint:tsc": "tsc --noEmit -p tsconfig.dev.json",
 		"lint:prettier": "prettier . --check",
+		"lint:sqlfluff": "sqlfluff lint src/database/",
 		"migrate": "ts-node src/scripts/migrate.ts | pino-pretty",
 		"start": "ts-node src/index.ts | pino-pretty",
 		"test": "npm run test:unit && npm run test:integration",

requirements.txt

@@ -0,0 +1 @@
+sqlfluff==3.2.5

src/database/initialization/application_form_field_to_json.sql

@@ -1,7 +1,9 @@
 SELECT drop_function('application_form_field_to_json');
 
-CREATE FUNCTION application_form_field_to_json(application_form_field application_form_fields)
-RETURNS JSONB AS $$
+CREATE FUNCTION application_form_field_to_json(
+  application_form_field application_form_fields
+)
+RETURNS jsonb AS $$
 DECLARE
   base_field_json JSONB;
 BEGIN

src/database/initialization/application_form_to_json.sql

@@ -1,7 +1,7 @@
 SELECT drop_function('application_form_to_json');
 
 CREATE FUNCTION application_form_to_json(application_form application_forms)
-RETURNS JSONB AS $$
+RETURNS jsonb AS $$
 DECLARE
   application_form_fields_json JSONB;
 BEGIN

src/database/initialization/base_field_localization_to_json.sql

@@ -1,7 +1,9 @@
 SELECT drop_function('base_field_localization_to_json');
 
-CREATE FUNCTION base_field_localization_to_json(base_field_localization base_field_localizations)
-RETURNS JSONB AS $$
+CREATE FUNCTION base_field_localization_to_json(
+  base_field_localization base_field_localizations
+)
+RETURNS jsonb AS $$
 BEGIN
   RETURN jsonb_build_object(
     'baseFieldId', base_field_localization.base_field_id,

src/database/initialization/base_field_to_json.sql

@@ -1,7 +1,7 @@
 SELECT drop_function('base_field_to_json');
 
 CREATE FUNCTION base_field_to_json(base_field base_fields)
-RETURNS JSONB AS $$
+RETURNS jsonb AS $$
 DECLARE
   localizations JSONB;
 BEGIN

src/database/initialization/bulk_upload_task_to_json.sql

@@ -1,7 +1,7 @@
 SELECT drop_function('bulk_upload_task_to_json');
 
 CREATE FUNCTION bulk_upload_task_to_json(bulk_upload_task bulk_upload_tasks)
-RETURNS JSONB AS $$
+RETURNS jsonb AS $$
 DECLARE
   source_json JSONB;
 BEGIN

src/database/initialization/changemaker_proposal_to_json.sql

@@ -1,7 +1,9 @@
 SELECT drop_function('changemaker_proposal_to_json');
 
-CREATE FUNCTION changemaker_proposal_to_json(changemaker_proposal changemakers_proposals)
-RETURNS JSONB AS $$
+CREATE FUNCTION changemaker_proposal_to_json(
+  changemaker_proposal changemakers_proposals
+)
+RETURNS jsonb AS $$
 DECLARE
   proposal_json JSONB;
   changemaker_json JSONB;

src/database/initialization/changemaker_to_json.sql

@@ -1,7 +1,10 @@
 SELECT drop_function('changemaker_to_json');
 
-CREATE FUNCTION changemaker_to_json(changemaker changemakers, keycloakUserId UUID DEFAULT NULL)
-RETURNS JSONB AS $$
+CREATE FUNCTION changemaker_to_json(
+  changemaker changemakers,
+  keycloakUserId uuid DEFAULT NULL
+)
+RETURNS jsonb AS $$
 DECLARE
   proposal_field_values_json JSONB;
 BEGIN

src/database/initialization/data_provider_to_json.sql

@@ -1,7 +1,7 @@
 SELECT drop_function('data_provider_to_json');
 
 CREATE FUNCTION data_provider_to_json(data_provider data_providers)
-RETURNS JSONB AS $$
+RETURNS jsonb AS $$
 BEGIN
   RETURN jsonb_build_object(
     'shortCode', data_provider.short_code,

src/database/initialization/funder_to_json.sql

@@ -1,7 +1,7 @@
 SELECT drop_function('funder_to_json');
 
 CREATE FUNCTION funder_to_json(funder funders)
-RETURNS JSONB AS $$
+RETURNS jsonb AS $$
 BEGIN
   RETURN jsonb_build_object(
     'shortCode', funder.short_code,

src/database/initialization/opportunity_to_json.sql

@@ -1,7 +1,7 @@
 SELECT drop_function('opportunity_to_json');
 
 CREATE FUNCTION opportunity_to_json(opportunity opportunities)
-RETURNS JSONB AS $$
+RETURNS jsonb AS $$
 BEGIN
   RETURN jsonb_build_object(
     'id', opportunity.id,

src/database/initialization/proposal_field_value_to_json.sql

@@ -1,7 +1,9 @@
 SELECT drop_function('proposal_field_value_to_json');
 
-CREATE FUNCTION proposal_field_value_to_json(proposal_field_value proposal_field_values)
-RETURNS JSONB AS $$
+CREATE FUNCTION proposal_field_value_to_json(
+  proposal_field_value proposal_field_values
+)
+RETURNS jsonb AS $$
 DECLARE
   application_form_field_json JSONB;
 BEGIN

src/database/initialization/proposal_to_json.sql

@@ -1,7 +1,7 @@
 SELECT drop_function('proposal_to_json');
 
 CREATE FUNCTION proposal_to_json(proposal proposals)
-RETURNS JSONB AS $$
+RETURNS jsonb AS $$
 DECLARE
   proposal_versions_json JSONB;
 BEGIN

src/database/initialization/proposal_version_to_json.sql

@@ -1,7 +1,7 @@
 SELECT drop_function('proposal_version_to_json');
 
 CREATE FUNCTION proposal_version_to_json(proposal_version proposal_versions)
-RETURNS JSONB AS $$
+RETURNS jsonb AS $$
 DECLARE
   proposal_field_values_json JSONB;
   source_json JSONB;

src/database/initialization/source_to_json.sql

@@ -1,7 +1,7 @@
 SELECT drop_function('source_to_json');
 
 CREATE FUNCTION source_to_json(source sources)
-RETURNS JSONB AS $$
+RETURNS jsonb AS $$
 DECLARE
   data_provider_json JSONB := NULL::JSONB;
   funder_json JSONB := NULL::JSONB;

src/database/initialization/user_changemaker_permission_to_json.sql

@@ -1,7 +1,9 @@
 SELECT drop_function('user_changemaker_permission_to_json');
 
-CREATE FUNCTION user_changemaker_permission_to_json(user_changemaker_permission user_changemaker_permissions)
-RETURNS JSONB AS $$
+CREATE FUNCTION user_changemaker_permission_to_json(
+  user_changemaker_permission user_changemaker_permissions
+)
+RETURNS jsonb AS $$
 BEGIN
   RETURN jsonb_build_object(
     'userKeycloakUserId', user_changemaker_permission.user_keycloak_user_id,

src/database/initialization/user_data_provider_permission_to_json.sql

@@ -1,7 +1,9 @@
 SELECT drop_function('user_data_provider_permission_to_json');
 
-CREATE FUNCTION user_data_provider_permission_to_json(user_data_provider_permission user_data_provider_permissions)
-RETURNS JSONB AS $$
+CREATE FUNCTION user_data_provider_permission_to_json(
+  user_data_provider_permission user_data_provider_permissions
+)
+RETURNS jsonb AS $$
 BEGIN
   RETURN jsonb_build_object(
     'userKeycloakUserId', user_data_provider_permission.user_keycloak_user_id,

src/database/initialization/user_funder_permission_to_json.sql

@@ -1,7 +1,9 @@
 SELECT drop_function('user_funder_permission_to_json');
 
-CREATE FUNCTION user_funder_permission_to_json(user_funder_permission user_funder_permissions)
-RETURNS JSONB AS $$
+CREATE FUNCTION user_funder_permission_to_json(
+  user_funder_permission user_funder_permissions
+)
+RETURNS jsonb AS $$
 BEGIN
   RETURN jsonb_build_object(
     'userKeycloakUserId', user_funder_permission.user_keycloak_user_id,

src/database/initialization/user_to_json.sql

@@ -1,7 +1,7 @@
 SELECT drop_function('user_to_json');
 
 CREATE FUNCTION user_to_json("user" users)
-RETURNS JSONB AS $$
+RETURNS jsonb AS $$
 DECLARE
   permissions_json JSONB := NULL::JSONB;
   user_changemaker_permissions_json JSONB := NULL::JSONB;

src/database/queries/applicationFormFields/insertOne.sql

@@ -9,4 +9,4 @@ INSERT INTO application_form_fields (
   :position,
   :label
 )
-RETURNING application_form_field_to_json(application_form_fields) AS "object";
+RETURNING application_form_field_to_json(application_form_fields) AS object;

src/database/queries/applicationFormFields/selectById.sql

@@ -1,3 +1,3 @@
-SELECT application_form_field_to_json(application_form_fields) AS "object"
+SELECT application_form_field_to_json(application_form_fields) AS object
 FROM application_form_fields
 WHERE id = :id;

src/database/queries/applicationFormFields/selectWithPagination.sql

@@ -1,12 +1,12 @@
-SELECT application_form_field_to_json(application_form_fields) AS "object"
+SELECT application_form_field_to_json(application_form_fields) AS object
 FROM application_form_fields
 WHERE
   CASE
     WHEN :applicationFormId::integer IS NULL THEN
-      true
+      TRUE
     ELSE
       application_form_id = :applicationFormId
     END
 ORDER BY position, id
 LIMIT :limit
-OFFSET :offset
+OFFSET :offset;

src/database/queries/applicationForms/insertOne.sql

@@ -3,13 +3,13 @@ INSERT INTO application_forms (
   version
 ) VALUES (
   :opportunityId,
-  COALESCE(
+  coalesce(
     (
-      SELECT MAX(af.version) + 1
-      FROM application_forms as af
+      SELECT max(af.version) + 1
+      FROM application_forms AS af
       WHERE af.opportunity_id = :opportunityId
     ),
     1
   )
 )
-RETURNING application_form_to_json(application_forms) AS "object";
+RETURNING application_form_to_json(application_forms) AS object;

src/database/queries/applicationForms/selectById.sql

@@ -1,4 +1,3 @@
-SELECT application_form_to_json(application_forms) AS "object"
+SELECT application_form_to_json(application_forms) AS object
 FROM application_forms
 WHERE id = :id;
-

src/database/queries/applicationForms/selectWithPagination.sql

@@ -1,5 +1,5 @@
-SELECT application_form_to_json(application_forms) AS "object"
+SELECT application_form_to_json(application_forms) AS object
 FROM application_forms
 ORDER BY id
 LIMIT :limit
-OFFSET :offset
+OFFSET :offset;

src/database/queries/baseFieldLocalizations/createOrUpdateByPrimaryKey.sql

@@ -11,6 +11,6 @@ INSERT INTO base_field_localizations (
 )
 ON CONFLICT (base_field_id, language)
 DO UPDATE SET
-  label = EXCLUDED.label,
-  description = EXCLUDED.description
-RETURNING base_field_localization_to_json(base_field_localizations) AS "object";
+  label = excluded.label,
+  description = excluded.description
+RETURNING base_field_localization_to_json(base_field_localizations) AS object;

src/database/queries/baseFieldLocalizations/selectAll.sql

@@ -1,2 +1,2 @@
-SELECT base_field_localization_to_json(base_field_localizations.*) as "object"
+SELECT base_field_localization_to_json(base_field_localizations.*) AS object
 FROM base_field_localizations;

src/database/queries/baseFieldLocalizations/selectWithPagination.sql

@@ -1,12 +1,12 @@
-SELECT base_field_localization_to_json(base_field_localizations) AS "object"
+SELECT base_field_localization_to_json(base_field_localizations) AS object
 FROM base_field_localizations
 WHERE
   CASE
     WHEN :baseFieldId::integer IS NULL THEN
-      true
+      TRUE
     ELSE
       base_field_id = :baseFieldId
     END
 ORDER BY created_at
 LIMIT :limit
-OFFSET :offset
+OFFSET :offset;