ci(tests): fuzz testing workflow for REST API

.circleci/config.yml

@@ -496,6 +496,7 @@ jobs:
       S3_REGION: '' # optional, defaults to 'us-east-1'
       ENCRYPTION_KEYS_PATH: 'test/assets/automate/encryptionKeys.json'
       FF_BILLING_INTEGRATION_ENABLED: 'true'
+      RATELIMITER_ENABLED: 'false'
     steps:
       - checkout
       - restore_cache:
@@ -578,6 +579,7 @@ jobs:
       S3_REGION: '' # optional, defaults to 'us-east-1'
       ENCRYPTION_KEYS_PATH: 'test/assets/automate/encryptionKeys.json'
       DISABLE_ALL_FFS: 'true'
+      RATELIMITER_ENABLED: 'false'
 
   test-server-multiregion:
     <<: *test-server-job
@@ -624,6 +626,7 @@ jobs:
       FF_WORKSPACES_MODULE_ENABLED: 'true'
       FF_WORKSPACES_MULTI_REGION_ENABLED: 'true'
       RUN_TESTS_IN_MULTIREGION_MODE: true
+      RATELIMITER_ENABLED: 'false'
 
   test-frontend-2:
     docker: &docker-node-browsers-image

.gitguardian.yml

@@ -29,5 +29,7 @@ secret:
       name: setup/keycloak/speckle-realm.json - secret for dev keycloak
     - match: 2e1b3675a4049cd39fe6db081735f747730969071528270800f00fa98720d198
       name: setup/keycloak/speckle-realm.json - algorithm name
+    - match: 164797a0ebc32f504e2dbaf48bec77969456bc301829039b34dc6787a1f5b0f3
+      name: setup/fuzzer/token.txt - fuzz test token
 
 version: 2

.github/workflows/rest-api-fuzzer.yml

@@ -0,0 +1,265 @@
+name: REST API Fuzz Test
+
+on:
+  workflow_dispatch:
+  # schedule:
+  #   - cron: "15 4 3 * *" # Run at 4:15am on the 3rd of every month
+  pull_request:
+    paths:
+      - '.github/workflows/rest-api-fuzzer.yml'
+      - 'setup/fuzzer/**/*'
+
+env:
+  BUILD_CONFIGURATION: Release
+  BUILD_PLATFORM: 'Any CPU'
+  RESTLER_VERSION: '9.2.4'
+  PYTHON_VERSION: '3.8'
+  DOTNET_VERSION: '6.0.x'
+
+jobs:
+  build-restler-fuzzer:
+    name: Fuzz test speckle-server REST API
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v4
+        name: Checkout speckle-server
+        with:
+          path: 'speckle-server'
+
+      - uses: actions/checkout@v4
+        name: Checkout RESTler Fuzzer
+        with:
+          repository: microsoft/restler-fuzzer
+          ref: v${{ env.RESTLER_VERSION }}
+          path: 'restler-fuzzer' # The path to clone the repository within the {{ github.workspace }} directory
+
+      - name: Setup .NET ${{ env.DOTNET_VERSION }}
+        uses: actions/setup-dotnet@v4
+        with:
+          dotnet-version: ${{ env.DOTNET_VERSION }}
+          cache: true
+          cache-dependency-path: ${{ github.workspace }}/restler-fuzzer/src/Restler.sln
+
+      - name: Restore NuGet packages
+        run: dotnet restore ${{ github.workspace }}/restler-fuzzer/src/Restler.sln
+
+      - name: Set up Python ${{ env.PYTHON_VERSION }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+          cache: 'pip'
+          cache-dependency-path: ${{ github.workspace }}/restler-fuzzer/restler/requirements.txt
+
+      - name: Install engine (Python) dependencies
+        run: |
+          pip install -r ${{ github.workspace }}/restler-fuzzer/restler/requirements.txt
+
+      # This doesn't currently work as it cannot find path to compiler
+      # - name: Restore cached Restler binaries
+      #   id: cache-restler-bin-restore
+      #   uses: actions/cache/restore@v4
+      #   with:
+      #     path: |
+      #       ${{ github.workspace }}/bin/restler
+      #     key: restler-binaries-${{ env.RESTLER_VERSION }}-${{ hashFiles('speckle-server/.github/workflows/rest-api-fuzzer.yml') }}
+
+      - name: Build RESTler
+        # if: steps.cache-restler-bin-restore.outputs.cache-hit != 'true'
+        run: |
+          python ${{ github.workspace }}/restler-fuzzer/build-restler.py --dest_dir ${{ github.workspace }}/bin
+          python -m compileall -b ${{ github.workspace }}/bin/restler
+
+      - name: Debug the built output
+        run: |
+          ls -la ${{ github.workspace }}/bin/restler
+          ls -la ${{ github.workspace }}/bin/restler/Restler
+
+      # - name: Save Restler binaries to cache
+      #   if: steps.cache-restler-bin-restore.outputs.cache-hit != 'true'
+      #   id: cache-restler-bin-save
+      #   uses: actions/cache/save@v4
+      #   with:
+      #     path: |
+      #       ${{ github.workspace }}/bin/restler
+      #     key: ${{ steps.cache-restler-bin-restore.outputs.cache-primary-key }}
+
+      - name: Restore cached Restler configuration
+        id: cache-config-restore
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            ${{ github.workspace }}/restlerConfig
+          key: restler-config-${{ hashFiles('speckle-server/setup/fuzzer/speckle-server.openapi.json') }}
+
+      - name: Generate RESTler config from OpenAPI specification
+        if: steps.cache-config-restore.outputs.cache-hit != 'true'
+        run: |
+          ${{ github.workspace }}/bin/restler/Restler generate_config --specs ${{ github.workspace }}/speckle-server/setup/fuzzer/speckle-server.openapi.json
+
+      - name: Print the Restler configuration
+        run: |
+          ls -la ${{ github.workspace }}
+          ls -la ${{ github.workspace }}/restlerConfig
+          echo ""
+          echo "############################################"
+          echo "# Engine settings                          #"
+          echo "# To customize, copy and save this file to #"
+          echo "# setup/fuzzer/settings.restler.json       #"
+          echo "############################################"
+          echo ""
+          cat ${{ github.workspace }}/restlerConfig/engine_settings.json
+          echo ""
+          echo "############################################"
+          echo "# Config                                   #"
+          echo "# To customize, copy and save this file to #"
+          echo "# setup/fuzzer/config.restler.json         #"
+          echo "############################################"
+          echo ""
+          cat ${{ github.workspace }}/restlerConfig/config.json
+          echo ""
+          echo "############################################"
+          echo "# Dictionary                               #"
+          echo "# To customize, copy and save this file to #"
+          echo "# setup/fuzzer/dictionary.restler.json     #"
+          echo "############################################"
+          echo ""
+          cat ${{ github.workspace }}/restlerConfig/dict.json
+          echo ""
+          echo "############################################"
+          echo "# Annotations                              #"
+          echo "# To customize, copy and save this file to #"
+          echo "# setup/fuzzer/annotations.restler.json    #"
+          echo "############################################"
+          echo ""
+          cat ${{ github.workspace }}/restlerConfig/annotations.json
+
+      - name: Save Restler Config
+        if: steps.cache-config-restore.outputs.cache-hit != 'true'
+        id: cache-config-save
+        uses: actions/cache/save@v4
+        with:
+          path: |
+            ${{ github.workspace }}/restlerConfig
+          key: ${{ steps.cache-config-restore.outputs.cache-primary-key }}
+
+      - name: Restore cached Restler grammar
+        id: cache-grammar-restore
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            ${{ github.workspace }}/Compile
+          key: restler-grammar-${{ hashFiles('speckle-server/setup/fuzzer/*.json') }}
+
+      - name: Generate RESTler grammar from Restler config
+        if: steps.cache-grammar-restore.outputs.cache-hit != 'true'
+        run: |
+          ${{ github.workspace }}/bin/restler/Restler compile ${{ github.workspace }}/speckle-server/setup/fuzzer/config.restler.json
+
+      - name: Print the contents of the Restler compile directory
+        if: always()
+        run: |
+          ls -la ${{ github.workspace }} || true
+          ls -la ${{ github.workspace }}/Compile || true
+          cat $(find ${{ github.workspace }}/Compile -type f -name "*.log") || true
+
+      - name: Save Grammar
+        if: steps.cache-grammar-restore.outputs.cache-hit != 'true'
+        id: cache-grammar-save
+        uses: actions/cache/save@v4
+        with:
+          path: |
+            ${{ github.workspace }}/Compile
+          key: ${{ steps.cache-grammar-restore.outputs.cache-primary-key }}
+
+      - name: Deploy dependencies (docker compose)
+        run: |
+          docker compose --file ${{ github.workspace }}/speckle-server/docker-compose-deps.yml up --detach
+
+      - name: Seed the database
+        run: |
+          sudo apt-get update
+          sudo apt-get install --yes --no-install-recommends postgresql-client
+          PGPASSWORD=speckle psql -h 127.0.0.1 -U speckle -d speckle -p 5432 -w < ${{ github.workspace }}/speckle-server/setup/fuzzer/speckle.backup.sql
+
+      - name: Deploy speckle-server (docker compose)
+        working-directory: ${{ github.workspace }}
+        timeout-minutes: 1
+        run: |
+          docker compose --file ${{ github.workspace }}/speckle-server/setup/fuzzer/docker-compose-speckle.yml up --detach
+
+      - name: Run RESTler coverage test
+        run: |
+          until curl --output /dev/null --silent --head --fail http://127.0.0.1:3000/readiness; do
+            echo "Waiting a further 3 seconds for speckle-server to start..."
+            sleep 3
+          done
+
+          ${{ github.workspace }}/bin/restler/Restler test \
+            --grammar_file "${{ github.workspace }}/Compile/grammar.py" \
+            --dictionary_file "${{ github.workspace }}/speckle-server/setup/fuzzer/dictionary.restler.json" \
+            --settings "${{ github.workspace }}/speckle-server/setup/fuzzer/settings.restler.json" \
+            --no_ssl \
+            --target_ip "127.0.0.1" \
+            --target_port "3000"
+
+      - name: Print the results
+        if: always()
+        run: |
+          ls -la ${{ github.workspace }}/Test
+          ls -la ${{ github.workspace }}/Test/RestlerResults || true
+          ls -la ${{ github.workspace }}/Test/ResponseBuckets || true
+          echo ""
+          echo "############################################"
+          echo "#            Engine stderr                 #"
+          echo "############################################"
+          echo ""
+          cat ${{ github.workspace }}/Test/EngineStdErr.txt || true
+          echo ""
+          echo "############################################"
+          echo "#     Engine stdout                        #"
+          echo "############################################"
+          echo ""
+          cat ${{ github.workspace }}/Test/EngineStdOut.txt || true
+          echo ""
+          echo "############################################"
+          echo "#     Results analyzer stderr              #"
+          echo "############################################"
+          echo ""
+          cat ${{ github.workspace }}/Test/ResultsAnalyzerStdErr.txt || true
+          echo ""
+          echo "############################################"
+          echo "#         Results analyzer stdout          #"
+          echo "############################################"
+          echo ""
+          cat ${{ github.workspace }}/Test/ResultsAnalyzerStdOut.txt || true
+          echo ""
+          echo "############################################"
+          echo "#    Coverage failures to investigate      #"
+          echo "############################################"
+          echo ""
+          cat ${{ github.workspace }}/Test/coverage_failures_to_investigate.txt || true
+          echo ""
+          echo "############################################"
+          echo "#              Restler logs                #"
+          echo "############################################"
+          echo ""
+          cat ${{ github.workspace }}/Test/restler-*.log || true
+
+      - uses: actions/upload-artifact@v4
+        name: Store the Test output
+        if: always()
+        with:
+          name: fuzz-test-rest-api-output
+          path: ${{ github.workspace }}/Test
+          if-no-files-found: error
+          retention-days: 5
+          overwrite: true
+
+      - name: Print Docker Compose logs
+        if: always()
+        run: |
+          docker compose --file ${{ github.workspace }}/speckle-server/docker-compose-deps.yml logs
+          docker compose --file ${{ github.workspace }}/speckle-server/setup/fuzzer/docker-compose-speckle.yml logs

packages/server/.env.test-example

@@ -6,4 +6,5 @@ PORT=0
 POSTGRES_URL=postgres://speckle:[email protected]/speckle2_test
 POSTGRES_USER=''
 MULTI_REGION_CONFIG_PATH="multiregion.test.json"
-#RUN_TESTS_IN_MULTIREGION_MODE=true
+#RUN_TESTS_IN_MULTIREGION_MODE=true
+RATELIMITER_ENABLED='false'

packages/server/modules/blobstorage/index.ts

@@ -129,10 +129,19 @@ export const init: SpeckleModule['init'] = async (app) => {
         uploadError?: Error | null | string
         formKey: string
       }>[] = []
-      const busboy = Busboy({
-        headers: req.headers,
-        limits: { fileSize: getFileSizeLimit() }
-      })
+      let busboy: Busboy.Busboy
+      try {
+        // Busboy does some validation of user input (headers) on creation
+        busboy = Busboy({
+          headers: req.headers,
+          limits: { fileSize: getFileSizeLimit() }
+        })
+      } catch (err) {
+        throw new BadRequestError(
+          err instanceof Error ? err.message : 'Error while uploading blob',
+          ensureError(err, 'Unknown error while uploading blob')
+        )
+      }
 
       const [projectDb, projectStorage] = await Promise.all([
         getProjectDbClient({ projectId: streamId }),

packages/server/modules/blobstorage/tests/blobstorage.integration.spec.js

@@ -227,4 +227,15 @@ describe('Blobs integration @blobstorage', () => {
 
     expect(response.status).to.equal(400)
   })
+
+  it('Returns 400 for missing content-type', async () => {
+    const streamId = await createStreamForTest()
+    const response = await request(app)
+      .post(`/api/stream/${streamId}/blob`)
+      .set('Authorization', `Bearer ${token}`)
+    // .set('Content-type', 'multipart/form-data; boundary=XXX') // purposefully missing content-type header
+
+    console.log(response.text)
+    expect(response.status).to.equal(400)
+  })
 })

packages/server/modules/core/services/ratelimiter.ts

@@ -2,7 +2,7 @@ import express from 'express'
 import {
   getRedisUrl,
   getIntFromEnv,
-  isTestEnv
+  getBooleanFromEnv
 } from '@/modules/shared/helpers/envHelper'
 import {
   BurstyRateLimiter,
@@ -55,6 +55,10 @@ export type RateLimiterMapping = {
 
 export type RateLimitAction = keyof typeof LIMITS
 
+export const isRateLimiterEnabled = (): boolean => {
+  return getBooleanFromEnv('RATELIMITER_ENABLED', true)
+}
+
 export const LIMITS = <const>{
   ALL_REQUESTS: {
     regularOptions: {
@@ -307,7 +311,7 @@ export const createRateLimiterMiddleware = (
     res: express.Response,
     next: express.NextFunction
   ) => {
-    if (isTestEnv()) return next()
+    if (!isRateLimiterEnabled()) return next()
     const path = getRequestPath(req) || ''
     const action = getActionForPath(path, req.method)
     const source = getSourceFromRequest(req)