Merge pull request #237 from cerberauth/make-discovery-faster

Make discovery faster

.github/workflows/ci.yml

@@ -40,7 +40,7 @@ jobs:
         run: go build -v ./...
 
       - name: Test
-        run: go test -race -coverprofile=coverage.out -covermode=atomic ./...
+        run: go test -coverprofile=coverage.out -covermode=atomic ./...
 
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v5

.github/workflows/scans.yml

@@ -20,6 +20,37 @@ permissions:
   packages: read
 
 jobs:
+  run-api-discovery:
+    name: API Discovery
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Run Server
+        run: docker run -d -p 8080:8080 ghcr.io/cerberauth/api-vulns-challenges/discoverable:latest
+
+      - name: Setup Go environment
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+
+      - name: VulnAPI
+        id: vulnapi
+        run: |
+          go run main.go discover api http://localhost:8080 --sqa-opt-out
+
+      - name: Stop Server
+        if: ${{ always() }}
+        run: docker stop $(docker ps -q --filter ancestor=ghcr.io/cerberauth/api-vulns-challenges/discoverable:latest)
+
   run-jwt-scans:
     name: JWT Scans
     runs-on: ubuntu-latest

internal/operation/operation.go

@@ -66,8 +66,9 @@ func getBody(body io.Reader) ([]byte, error) {
 }
 
 func NewOperation(method string, operationUrl string, body io.Reader, client *request.Client) (*Operation, error) {
-	if client == nil {
-		client = request.GetDefaultClient()
+	operationClient := client
+	if operationClient == nil {
+		operationClient = request.GetDefaultClient()
 	}
 
 	parsedUrl, err := url.Parse(operationUrl)
@@ -81,7 +82,7 @@ func NewOperation(method string, operationUrl string, body io.Reader, client *re
 	}
 
 	return &Operation{
-		Client: client,
+		Client: operationClient,
 
 		Method:          method,
 		URL:             *parsedUrl,

scan/discover/utils.go

@@ -24,26 +24,50 @@ func ExtractBaseURL(inputURL *url.URL) *url.URL {
 
 func ScanURLs(scanUrls []string, op *operation.Operation, securityScheme *auth.SecurityScheme, r *report.ScanReport, vulnReport *report.IssueReport) (*report.ScanReport, error) {
 	securitySchemes := []*auth.SecurityScheme{securityScheme}
-
 	base := ExtractBaseURL(&op.URL)
-	for _, path := range scanUrls {
-		newOperation, err := operation.NewOperation(http.MethodGet, base.ResolveReference(&url.URL{Path: path}).String(), nil, op.Client)
-		newOperation.SetSecuritySchemes(securitySchemes)
-		if err != nil {
-			return r, err
-		}
+	chunkSize := 20
+	results := make(chan *scan.IssueScanAttempt, len(scanUrls))
+	errors := make(chan error, len(scanUrls))
 
-		attempt, err := scan.ScanURL(newOperation, securityScheme)
-		if err != nil {
-			return r, err
+	for i := 0; i < len(scanUrls); i += chunkSize {
+		end := i + chunkSize
+		if end > len(scanUrls) {
+			end = len(scanUrls)
 		}
+		chunk := scanUrls[i:end]
+
+		go func(chunk []string) {
+			for _, path := range chunk {
+				newOperation, err := operation.NewOperation(http.MethodGet, base.ResolveReference(&url.URL{Path: path}).String(), nil, op.Client)
+				newOperation.SetSecuritySchemes(securitySchemes)
+				if err != nil {
+					errors <- err
+					return
+				}
 
-		r.AddScanAttempt(attempt)
-		if attempt.Response.GetStatusCode() == http.StatusOK { // TODO: check if the response contains the expected content
-			r.WithData(DiscoverData{
-				URL: attempt.Request.GetURL(),
-			}).AddIssueReport(vulnReport.Fail()).End()
-			return r, nil
+				attempt, err := scan.ScanURL(newOperation, securityScheme)
+				if err != nil {
+					errors <- err
+					return
+				}
+
+				results <- attempt
+			}
+		}(chunk)
+	}
+
+	for i := 0; i < len(scanUrls); i++ {
+		select {
+		case attempt := <-results:
+			r.AddScanAttempt(attempt)
+			if attempt.Response.GetStatusCode() == http.StatusOK { // TODO: check if the response contains the expected content
+				r.WithData(DiscoverData{
+					URL: attempt.Request.GetURL(),
+				}).AddIssueReport(vulnReport.Fail()).End()
+				return r, nil
+			}
+		case err := <-errors:
+			return r, err
 		}
 	}
 

scenario/discover_domain.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"net"
 	"net/http"
+	"sync"
 
 	"github.com/cerberauth/vulnapi/internal/operation"
 	"github.com/cerberauth/vulnapi/internal/request"
@@ -17,43 +18,62 @@ var commonHostnames = []string{"www", "api", "graphql", "graph", "app", "auth",
 
 func searchByCommonHostnames(domain string) []string {
 	subdomains := []string{}
+	var mu sync.Mutex
+	var wg sync.WaitGroup
 
 	for _, hostname := range commonHostnames {
-		subdomain := hostname + "." + domain
-		_, err := net.LookupIP(subdomain)
-		if err != nil {
-			continue
-		}
+		wg.Add(1)
+		go func(hostname string) {
+			defer wg.Done()
+			subdomain := hostname + "." + domain
+			_, err := net.LookupIP(subdomain)
+			if err != nil {
+				return
+			}
 
-		subdomains = append(subdomains, subdomain)
+			mu.Lock()
+			subdomains = append(subdomains, subdomain)
+			mu.Unlock()
+		}(hostname)
 	}
 
+	wg.Wait()
 	return subdomains
 }
 
 func searchByLookupIP(rootDomain string) ([]string, error) {
 	subdomains := []string{}
+	var mu sync.Mutex
+	var wg sync.WaitGroup
 
 	ips, err := net.LookupIP(rootDomain)
 	if err != nil {
 		return subdomains, err
 	}
 
 	for _, ip := range ips {
-		hosts, err := net.LookupAddr(ip.String())
-		if err != nil {
-			continue
-		}
+		wg.Add(1)
+		go func(ip net.IP) {
+			defer wg.Done()
+			hosts, err := net.LookupAddr(ip.String())
+			if err != nil {
+				return
+			}
 
-		for _, host := range hosts {
-			if len(host) == 0 {
-				continue
+			for _, host := range hosts {
+				if len(host) == 0 {
+					continue
+				}
+				subdomain := host[:len(host)-1] // Remove the trailing dot
+
+				mu.Lock()
+				subdomains = append(subdomains, subdomain)
+				mu.Unlock()
 			}
-			subdomain := host[:len(host)-1] // Remove the trailing dot
-			subdomains = append(subdomains, subdomain)
-		}
+		}(ip)
 	}
 
+	wg.Wait()
 	return subdomains, nil
 }
 

seclist/lists/graphql.txt

@@ -2,11 +2,6 @@ ___graphql
 altair
 explorer
 graphiql
-graphiql.css
-graphiql/finland
-graphiql.js
-graphiql.min.css
-graphiql.min.js
 graphiql.php
 graphql
 graphql/console
@@ -16,18 +11,11 @@ graphql/schema.json
 graphql/schema.xml
 graphql/schema.yaml
 playground
-subscriptions
 api/graphql
-je/graphql
 graph
 v1/altair
 v1/explorer
 v1/graphiql
-v1/graphiql.css
-v1/graphiql/finland
-v1/graphiql.js
-v1/graphiql.min.css
-v1/graphiql.min.js
 v1/graphiql.php
 v1/graphql
 v1/graphql/console
@@ -43,11 +31,6 @@ v1/graph
 v2/altair
 v2/explorer
 v2/graphiql
-v2/graphiql.css
-v2/graphiql/finland
-v2/graphiql.js
-v2/graphiql.min.css
-v2/graphiql.min.js
 v2/graphiql.php
 v2/graphql
 v2/graphql/console
@@ -59,44 +42,4 @@ v2/graphql/schema.yaml
 v2/playground
 v2/subscriptions
 v2/api/graphql
-v2/graph
-v3/altair
-v3/explorer
-v3/graphiql
-v3/graphiql.css
-v3/graphiql/finland
-v3/graphiql.js
-v3/graphiql.min.css
-v3/graphiql.min.js
-v3/graphiql.php
-v3/graphql
-v3/graphql/console
-v3/graphql-explorer
-v3/graphql.php
-v3/graphql/schema.json
-v3/graphql/schema.xml
-v3/graphql/schema.yaml
-v3/playground
-v3/subscriptions
-v3/api/graphql
-v3/graph
-v4/altair
-v4/explorer
-v4/graphiql
-v4/graphiql.css
-v4/graphiql/finland
-v4/graphiql.js
-v4/graphiql.min.css
-v4/graphiql.min.js
-v4/graphiql.php
-v4/graphql
-v4/graphql/console
-v4/graphql-explorer
-v4/graphql.php
-v4/graphql/schema.json
-v4/graphql/schema.xml
-v4/graphql/schema.yaml
-v4/playground
-v4/subscriptions
-v4/api/graphql
-v4/graph
+v2/graph

seclist/lists/swagger.txt

@@ -44,7 +44,6 @@ api/v2/swagger.json
 api/v2/swagger-ui/
 api/v2/swagger-ui.html
 api/v2/swagger-ui.json
-graphql
 api
 api/v1/
 api/v2