CHANGE to sample method for iterator.

src/actionruntime/common/utils.go

@@ -23,8 +23,8 @@ import (
 	parser_template "github.com/illacloud/builder-backend/src/utils/parser/template"
 )
 
-const SQL_RESULT_MEMORY_LIMIT = 524288000   // 500 * 1024 * 1024 bytes
-const SQL_RESULT_MEMORY_CHECK_RATE = 100000 // check in every 100,000 times
+const SQL_RESULT_MEMORY_LIMIT = 209715200   // 200 * 1024 * 1024 bytes
+const SQL_RESULT_MEMORY_CHECK_SAMPLE = 1000 // check 1000 item bytes and calculate max item capacity
 
 func RetrieveToMap(rows *sql.Rows) ([]map[string]interface{}, error) {
 	columns, err := rows.Columns()

src/actionruntime/postgresql/base.go

@@ -21,6 +21,7 @@ import (
 	"encoding/pem"
 	"errors"
 	"fmt"
+	"log"
 	"net/url"
 	"reflect"
 
@@ -171,6 +172,7 @@ func RetrieveToMap(rows pgx.Rows) ([]map[string]interface{}, error) {
 	valuePtrs := make([]interface{}, count)
 
 	iteratorNums := 0
+	tableDataCapacity := 10000
 	for rows.Next() {
 		iteratorNums++
 		for i := 0; i < count; i++ {
@@ -195,12 +197,14 @@ func RetrieveToMap(rows pgx.Rows) ([]map[string]interface{}, error) {
 
 		tableData = append(tableData, entry)
 
-		// check tableData size every 100 results
-		if iteratorNums%common.SQL_RESULT_MEMORY_CHECK_RATE == 0 {
-			tableDataSize := size.Of(tableData)
-			if tableDataSize > common.SQL_RESULT_MEMORY_LIMIT {
-				return nil, errors.New("returned result exceeds 500MiB, please adjust the query limit to reduce the number of results")
-			}
+		// check tableData size by sample
+		if iteratorNums == common.SQL_RESULT_MEMORY_CHECK_SAMPLE {
+			tableDataSizeBySample := size.Of(tableData)
+			tableDataCapacity = (common.SQL_RESULT_MEMORY_LIMIT / tableDataSizeBySample) * 1000
+		}
+		if iteratorNums > tableDataCapacity {
+			log.Printf("[ERROR] RetrieveToMap result exceeds 200MiB by iteratorNums: %d", iteratorNums)
+			return nil, errors.New("returned result exceeds 200MiB, please adjust the query limit to reduce the number of results")
 		}
 	}